@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9841d48a10
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Use TGI-like incremental detokenization (#984)

Antoni Baum committed Sep 13, 2023 at 13:38 UTC 9841d48a108b0e14da6572de6be0b47d70bbb641
4 files changed +139 -38
tests/engine/test_detokenize.py new
+55
@@ -0,0 +1,55 @@
1 +import pytest
2 +
3 +from transformers import AutoTokenizer
4 +
5 +from vllm.transformers_utils.tokenizer import detokenize_incrementally
6 +
7 +TRUTH = [
8 + "Hello here, this is a simple test",
9 + "vLLM is a high-throughput and memory-efficient inference and serving engine for LLMs. It is designed to be used in production environments, where inference and serving",
10 + "我很感谢你的热情"
11 +]
12 +TOKENIZERS = [
13 + "facebook/opt-125m",
14 + "gpt2",
15 + "bigcode/tiny_starcoder_py",
16 + "EleutherAI/gpt-j-6b",
17 + "EleutherAI/pythia-70m",
18 + "bigscience/bloom-560m",
19 + "mosaicml/mpt-7b",
20 + "tiiuae/falcon-7b",
21 + "meta-llama/Llama-2-7b-hf",
22 + "codellama/CodeLlama-7b-hf",
23 +]
24 +
25 +
26 +def _run_incremental_decode(tokenizer, all_input_ids):
27 + decoded_text = ""
28 + offset = 0
29 + token_offset = 0
30 + prev_tokens = None
31 + for i in range(len(all_input_ids)):
32 + new_tokens, text, offset, token_offset = detokenize_incrementally(
33 + tokenizer,
34 + all_input_ids[:i + 1],
35 + prev_tokens,
36 + offset,
37 + token_offset,
38 + skip_special_tokens=False)
39 + decoded_text += text
40 + if prev_tokens is None:
41 + prev_tokens = new_tokens
42 + else:
43 + prev_tokens += new_tokens
44 + return decoded_text
45 +
46 +
47 +@pytest.mark.parametrize("truth", TRUTH)
48 +@pytest.mark.parametrize("tokenizer_id", TOKENIZERS)
49 +def test_decode_streaming(tokenizer_id, truth):
50 + tokenizer = AutoTokenizer.from_pretrained(tokenizer_id)
51 + all_input_ids = tokenizer(truth, add_special_tokens=False)["input_ids"]
52 +
53 + decoded_text = _run_incremental_decode(tokenizer, all_input_ids)
54 +
55 + assert decoded_text == truth
vllm/engine/llm_engine.py
+16 -9
@@ -623,15 +623,22 @@ class LLMEngine:
623
624 def _decode_sequence(self, seq: Sequence) -> None:
625 """Decodes the new token for a sequence."""
626 - new_token, new_output_text = detokenize_incrementally(
627 - self.tokenizer,
628 - seq.output_tokens,
629 - seq.get_last_token_id(),
630 - skip_special_tokens=True,
631 - )
632 - if new_token is not None:
633 - seq.output_tokens.append(new_token)
634 - seq.output_text = new_output_text
626 + (new_tokens, new_output_text, prefix_offset,
627 + read_offset) = detokenize_incrementally(
628 + self.tokenizer,
629 + all_input_ids=seq.get_token_ids(),
630 + prev_tokens=seq.tokens,
631 + prefix_offset=seq.prefix_offset,
632 + read_offset=seq.read_offset,
633 + skip_special_tokens=True,
634 + )
635 + if seq.tokens is None:
636 + seq.tokens = new_tokens
637 + else:
638 + seq.tokens.extend(new_tokens)
639 + seq.prefix_offset = prefix_offset
640 + seq.read_offset = read_offset
641 + seq.output_text += new_output_text
642
643 def _check_stop(self, seq: Sequence,
644 sampling_params: SamplingParams) -> None:
vllm/sequence.py
+6 -1
@@ -114,7 +114,6 @@ class Sequence:
114
115 self.data = SequenceData(prompt_token_ids)
116 self.output_logprobs: List[Dict[int, float]] = []
117 - self.output_tokens: List[str] = []
117 self.output_text = ""
118
119 self.logical_token_blocks: List[LogicalTokenBlock] = []
@@ -122,6 +121,12 @@ class Sequence:
121 self._append_tokens_to_blocks(prompt_token_ids)
122 self.status = SequenceStatus.WAITING
123
124 + # Used for incremental detokenization
125 + self.prefix_offset = 0
126 + self.read_offset = 0
127 + # Input + output tokens
128 + self.tokens: Optional[List[str]] = None
129 +
130 def _append_logical_block(self) -> None:
131 block = LogicalTokenBlock(
132 block_number=len(self.logical_token_blocks),
vllm/transformers_utils/tokenizer.py
+62 -28
@@ -1,4 +1,4 @@
1 -from typing import List, Tuple, Union
1 +from typing import List, Optional, Tuple, Union
2
3 from transformers import (AutoTokenizer, PreTrainedTokenizer,
4 PreTrainedTokenizerFast)
@@ -67,33 +67,11 @@ def get_tokenizer(
67 return tokenizer
68
69
70 -def detokenize_incrementally(
70 +def _convert_tokens_to_string_with_added_encoders(
71 tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
72 - prev_output_tokens: List[str],
73 - new_token_id: int,
72 + output_tokens: List[str],
73 skip_special_tokens: bool,
75 -) -> Tuple[str, str]:
76 - """Detokenizes the new token in conjunction with the previous output tokens.
77 -
78 - NOTE: This function does not update prev_output_tokens.
79 -
80 - Returns:
81 - new_token: The new token as a string.
82 - output_text: The new output text as a string.
83 - """
84 - if skip_special_tokens and (new_token_id in tokenizer.all_special_ids):
85 - return None, prev_output_tokens
86 - new_token = tokenizer.convert_ids_to_tokens(
87 - new_token_id, skip_special_tokens=skip_special_tokens)
88 - output_tokens = prev_output_tokens + [new_token]
89 -
90 - # Convert the tokens to a string.
91 - # Optimization: If the tokenizer does not have `added_tokens_encoder`,
92 - # then we can directly use `convert_tokens_to_string`.
93 - if not getattr(tokenizer, "added_tokens_encoder", {}):
94 - output_text = tokenizer.convert_tokens_to_string(output_tokens)
95 - return new_token, output_text
96 -
74 +) -> str:
75 # Adapted from
76 # https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/tokenization_utils.py#L921
77 # NOTE(woosuk): The following code is slow because it runs a for loop over
@@ -115,5 +93,61 @@ def detokenize_incrementally(
93 if current_sub_text:
94 sub_text = tokenizer.convert_tokens_to_string(current_sub_text)
95 sub_texts.append(sub_text)
118 - output_text = " ".join(sub_texts)
119 - return new_token, output_text
96 + return " ".join(sub_texts)
97 +
98 +
99 +# Based on
100 +# https://github.com/huggingface/text-generation-inference/blob/v0.9.4/server/text_generation_server/models/model.py#L62C9-L62C15
101 +# under Apache 2.0 license
102 +def detokenize_incrementally(
103 + tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
104 + all_input_ids: List[int],
105 + prev_tokens: Optional[List[str]],
106 + prefix_offset: int = 0,
107 + read_offset: int = 0,
108 + skip_special_tokens: bool = False,
109 +) -> Tuple[List[str], str, int, int]:
110 + new_token_id = all_input_ids[-1]
111 + # This is the first iteration for this sequence
112 + if prev_tokens is None:
113 + new_tokens = tokenizer.convert_ids_to_tokens(
114 + all_input_ids, skip_special_tokens=skip_special_tokens)
115 + output_tokens = new_tokens
116 + # 5 is an arbitrary value that should work for all
117 + # tokenizers (bigger = more conservative).
118 + # Subtract 1 extra to account for the generated token.
119 + prefix_offset = max(len(output_tokens) - 6, 0)
120 + read_offset = max(len(output_tokens) - 1, 0)
121 + else:
122 + new_token = tokenizer.convert_ids_to_tokens(
123 + new_token_id, skip_special_tokens=skip_special_tokens)
124 + new_tokens = [new_token]
125 + output_tokens = prev_tokens + new_tokens
126 +
127 + # The prefix text is necessary only to defeat cleanup algorithms in
128 + # the decode which decide to add a space or not depending on the
129 + # surrounding ids.
130 + if not getattr(tokenizer, "added_tokens_encoder", {}):
131 + prefix_text = tokenizer.convert_tokens_to_string(
132 + output_tokens[prefix_offset:read_offset])
133 + new_text = tokenizer.convert_tokens_to_string(
134 + output_tokens[prefix_offset:])
135 + else:
136 + prefix_text = _convert_tokens_to_string_with_added_encoders(
137 + tokenizer,
138 + output_tokens[prefix_offset:read_offset],
139 + skip_special_tokens=skip_special_tokens)
140 + new_text = _convert_tokens_to_string_with_added_encoders(
141 + tokenizer,
142 + output_tokens[prefix_offset:],
143 + skip_special_tokens=skip_special_tokens)
144 +
145 + if len(new_text) > len(prefix_text) and not new_text.endswith("�"):
146 + # utf-8 char at the end means it's a potential unfinished byte sequence
147 + # from byte fallback tokenization.
148 + # If it's in the middle, it's probably a real invalid id generated
149 + # by the model
150 + new_text = new_text[len(prefix_text):]
151 + return new_tokens, new_text, read_offset, len(output_tokens)
152 + else:
153 + return new_tokens, "", prefix_offset, read_offset