A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Use TGI-like incremental detokenization (#984)
Antoni Baum committed
Sep 13, 2023 at 13:38 UTC
9841d48a108b0e14da6572de6be0b47d70bbb641
4 files changed
+139
-38
tests/engine/test_detokenize.py
new
+55
@@ -0,0 +1,55 @@
1
+import pytest
2
+
3
+from transformers import AutoTokenizer
4
+
5
+from vllm.transformers_utils.tokenizer import detokenize_incrementally
6
+
7
+TRUTH = [
8
+ "Hello here, this is a simple test",
9
+ "vLLM is a high-throughput and memory-efficient inference and serving engine for LLMs. It is designed to be used in production environments, where inference and serving",
10
+ "我很感谢你的热情"
11
+]
12
+TOKENIZERS = [
13
+ "facebook/opt-125m",
14
+ "gpt2",
15
+ "bigcode/tiny_starcoder_py",
16
+ "EleutherAI/gpt-j-6b",
17
+ "EleutherAI/pythia-70m",
18
+ "bigscience/bloom-560m",
19
+ "mosaicml/mpt-7b",
20
+ "tiiuae/falcon-7b",
21
+ "meta-llama/Llama-2-7b-hf",
22
+ "codellama/CodeLlama-7b-hf",
23
+]
24
+
25
+
26
+def _run_incremental_decode(tokenizer, all_input_ids):
27
+ decoded_text = ""
28
+ offset = 0
29
+ token_offset = 0
30
+ prev_tokens = None
31
+ for i in range(len(all_input_ids)):
32
+ new_tokens, text, offset, token_offset = detokenize_incrementally(
33
+ tokenizer,
34
+ all_input_ids[:i + 1],
35
+ prev_tokens,
36
+ offset,
37
+ token_offset,
38
+ skip_special_tokens=False)
39
+ decoded_text += text
40
+ if prev_tokens is None:
41
+ prev_tokens = new_tokens
42
+ else:
43
+ prev_tokens += new_tokens
44
+ return decoded_text
45
+
46
+
47
+@pytest.mark.parametrize("truth", TRUTH)
48
+@pytest.mark.parametrize("tokenizer_id", TOKENIZERS)
49
+def test_decode_streaming(tokenizer_id, truth):
50
+ tokenizer = AutoTokenizer.from_pretrained(tokenizer_id)
51
+ all_input_ids = tokenizer(truth, add_special_tokens=False)["input_ids"]
52
+
53
+ decoded_text = _run_incremental_decode(tokenizer, all_input_ids)
54
+
55
+ assert decoded_text == truth
vllm/engine/llm_engine.py
+16
-9
@@ -623,15 +623,22 @@ class LLMEngine:
623
624
def _decode_sequence(self, seq: Sequence) -> None:
625
"""Decodes the new token for a sequence."""
626
- new_token, new_output_text = detokenize_incrementally(
627
- self.tokenizer,
628
- seq.output_tokens,
629
- seq.get_last_token_id(),
630
- skip_special_tokens=True,
631
- )
632
- if new_token is not None:
633
- seq.output_tokens.append(new_token)
634
- seq.output_text = new_output_text
626
+ (new_tokens, new_output_text, prefix_offset,
627
+ read_offset) = detokenize_incrementally(
628
+ self.tokenizer,
629
+ all_input_ids=seq.get_token_ids(),
630
+ prev_tokens=seq.tokens,
631
+ prefix_offset=seq.prefix_offset,
632
+ read_offset=seq.read_offset,
633
+ skip_special_tokens=True,
634
+ )
635
+ if seq.tokens is None:
636
+ seq.tokens = new_tokens
637
+ else:
638
+ seq.tokens.extend(new_tokens)
639
+ seq.prefix_offset = prefix_offset
640
+ seq.read_offset = read_offset
641
+ seq.output_text += new_output_text
642
643
def _check_stop(self, seq: Sequence,
644
sampling_params: SamplingParams) -> None:
vllm/sequence.py
+6
-1
@@ -114,7 +114,6 @@ class Sequence:
114
115
self.data = SequenceData(prompt_token_ids)
116
self.output_logprobs: List[Dict[int, float]] = []
117
- self.output_tokens: List[str] = []
117
self.output_text = ""
118
119
self.logical_token_blocks: List[LogicalTokenBlock] = []
@@ -122,6 +121,12 @@ class Sequence:
121
self._append_tokens_to_blocks(prompt_token_ids)
122
self.status = SequenceStatus.WAITING
123
124
+ # Used for incremental detokenization
125
+ self.prefix_offset = 0
126
+ self.read_offset = 0
127
+ # Input + output tokens
128
+ self.tokens: Optional[List[str]] = None
129
+
130
def _append_logical_block(self) -> None:
131
block = LogicalTokenBlock(
132
block_number=len(self.logical_token_blocks),
vllm/transformers_utils/tokenizer.py
+62
-28
@@ -1,4 +1,4 @@
1
-from typing import List, Tuple, Union
1
+from typing import List, Optional, Tuple, Union
2
3
from transformers import (AutoTokenizer, PreTrainedTokenizer,
4
PreTrainedTokenizerFast)
@@ -67,33 +67,11 @@ def get_tokenizer(
67
return tokenizer
68
69
70
-def detokenize_incrementally(
70
+def _convert_tokens_to_string_with_added_encoders(
71
tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
72
- prev_output_tokens: List[str],
73
- new_token_id: int,
72
+ output_tokens: List[str],
73
skip_special_tokens: bool,
75
-) -> Tuple[str, str]:
76
- """Detokenizes the new token in conjunction with the previous output tokens.
77
-
78
- NOTE: This function does not update prev_output_tokens.
79
-
80
- Returns:
81
- new_token: The new token as a string.
82
- output_text: The new output text as a string.
83
- """
84
- if skip_special_tokens and (new_token_id in tokenizer.all_special_ids):
85
- return None, prev_output_tokens
86
- new_token = tokenizer.convert_ids_to_tokens(
87
- new_token_id, skip_special_tokens=skip_special_tokens)
88
- output_tokens = prev_output_tokens + [new_token]
89
-
90
- # Convert the tokens to a string.
91
- # Optimization: If the tokenizer does not have `added_tokens_encoder`,
92
- # then we can directly use `convert_tokens_to_string`.
93
- if not getattr(tokenizer, "added_tokens_encoder", {}):
94
- output_text = tokenizer.convert_tokens_to_string(output_tokens)
95
- return new_token, output_text
96
-
74
+) -> str:
75
# Adapted from
76
# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/tokenization_utils.py#L921
77
# NOTE(woosuk): The following code is slow because it runs a for loop over
@@ -115,5 +93,61 @@ def detokenize_incrementally(
93
if current_sub_text:
94
sub_text = tokenizer.convert_tokens_to_string(current_sub_text)
95
sub_texts.append(sub_text)
118
- output_text = " ".join(sub_texts)
119
- return new_token, output_text
96
+ return " ".join(sub_texts)
97
+
98
+
99
+# Based on
100
+# https://github.com/huggingface/text-generation-inference/blob/v0.9.4/server/text_generation_server/models/model.py#L62C9-L62C15
101
+# under Apache 2.0 license
102
+def detokenize_incrementally(
103
+ tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
104
+ all_input_ids: List[int],
105
+ prev_tokens: Optional[List[str]],
106
+ prefix_offset: int = 0,
107
+ read_offset: int = 0,
108
+ skip_special_tokens: bool = False,
109
+) -> Tuple[List[str], str, int, int]:
110
+ new_token_id = all_input_ids[-1]
111
+ # This is the first iteration for this sequence
112
+ if prev_tokens is None:
113
+ new_tokens = tokenizer.convert_ids_to_tokens(
114
+ all_input_ids, skip_special_tokens=skip_special_tokens)
115
+ output_tokens = new_tokens
116
+ # 5 is an arbitrary value that should work for all
117
+ # tokenizers (bigger = more conservative).
118
+ # Subtract 1 extra to account for the generated token.
119
+ prefix_offset = max(len(output_tokens) - 6, 0)
120
+ read_offset = max(len(output_tokens) - 1, 0)
121
+ else:
122
+ new_token = tokenizer.convert_ids_to_tokens(
123
+ new_token_id, skip_special_tokens=skip_special_tokens)
124
+ new_tokens = [new_token]
125
+ output_tokens = prev_tokens + new_tokens
126
+
127
+ # The prefix text is necessary only to defeat cleanup algorithms in
128
+ # the decode which decide to add a space or not depending on the
129
+ # surrounding ids.
130
+ if not getattr(tokenizer, "added_tokens_encoder", {}):
131
+ prefix_text = tokenizer.convert_tokens_to_string(
132
+ output_tokens[prefix_offset:read_offset])
133
+ new_text = tokenizer.convert_tokens_to_string(
134
+ output_tokens[prefix_offset:])
135
+ else:
136
+ prefix_text = _convert_tokens_to_string_with_added_encoders(
137
+ tokenizer,
138
+ output_tokens[prefix_offset:read_offset],
139
+ skip_special_tokens=skip_special_tokens)
140
+ new_text = _convert_tokens_to_string_with_added_encoders(
141
+ tokenizer,
142
+ output_tokens[prefix_offset:],
143
+ skip_special_tokens=skip_special_tokens)
144
+
145
+ if len(new_text) > len(prefix_text) and not new_text.endswith("�"):
146
+ # utf-8 char at the end means it's a potential unfinished byte sequence
147
+ # from byte fallback tokenization.
148
+ # If it's in the middle, it's probably a real invalid id generated
149
+ # by the model
150
+ new_text = new_text[len(prefix_text):]
151
+ return new_tokens, new_text, read_offset, len(output_tokens)
152
+ else:
153
+ return new_tokens, "", prefix_offset, read_offset