A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
[Core] Improve detokenization performance for prefill (#3469)
Co-authored-by: MeloYang <meloyang05@gmail.com>
Antoni Baum committed
Mar 22, 2024 at 13:44 UTC
bfdb1ba5c3fb14387c69acb1f5067102d8028e56
4 files changed
+385
-89
tests/tokenization/test_detokenize.py
+151
-12
@@ -1,13 +1,17 @@
1
import pytest
2
3
from transformers import AutoTokenizer
4
+from typing import List, Dict
5
6
+from vllm.sequence import Sequence, Logprob, SamplingParams, SequenceGroup
7
+from vllm.transformers_utils.tokenizer_group import get_tokenizer_group
8
from vllm.transformers_utils.tokenizer import detokenize_incrementally
9
+from vllm.transformers_utils.detokenizer import Detokenizer
10
11
TRUTH = [
8
- "Hello here, this is a simple test", # noqa: E501
9
- "vLLM is a high-throughput and memory-efficient inference and serving engine for LLMs. It is designed to be used in production environments, where inference and serving", # noqa: E501
10
- "我很感谢你的热情" # noqa: E501
12
+ "Hello here, this is a simple test",
13
+ "vLLM is a high-throughput and memory-efficient inference and serving engine for LLMs. It is designed to be used in production environments, where inference and serving", # noqa
14
+ "我很感谢你的热情"
15
]
16
TOKENIZERS = [
17
"facebook/opt-125m",
@@ -24,12 +28,12 @@ TOKENIZERS = [
28
29
30
def _run_incremental_decode(tokenizer, all_input_ids,
27
- skip_special_tokens: bool):
31
+ skip_special_tokens: bool, starting_index: int):
32
decoded_text = ""
33
offset = 0
34
token_offset = 0
35
prev_tokens = None
32
- for i in range(len(all_input_ids)):
36
+ for i in range(starting_index, len(all_input_ids)):
37
new_tokens, text, offset, token_offset = detokenize_incrementally(
38
tokenizer,
39
all_input_ids[:i + 1],
@@ -46,17 +50,152 @@ def _run_incremental_decode(tokenizer, all_input_ids,
50
51
52
@pytest.mark.parametrize("truth", TRUTH)
53
+@pytest.mark.parametrize("with_prompt", [True, False])
54
@pytest.mark.parametrize("tokenizer_id", TOKENIZERS)
55
@pytest.mark.parametrize("skip_special_tokens", (True, False))
51
-def test_decode_streaming(tokenizer_id, truth, skip_special_tokens):
56
+def test_decode_streaming(tokenizer_id, truth, with_prompt,
57
+ skip_special_tokens):
58
tokenizer = AutoTokenizer.from_pretrained(tokenizer_id)
53
- all_input_ids = tokenizer(truth, add_special_tokens=False)["input_ids"]
59
+ if with_prompt:
60
+ truth_tokens = tokenizer(truth, add_special_tokens=False)["input_ids"]
61
+ prompt_input_ids = truth_tokens[:len(truth) // 2]
62
+ generated_input_ids = truth_tokens[len(truth) // 2:]
63
+ all_input_ids = prompt_input_ids + generated_input_ids
64
+ starting_index = len(prompt_input_ids)
65
+ prompt = tokenizer.decode(prompt_input_ids,
66
+ skip_special_tokens=skip_special_tokens)
67
+ generated = truth[len(prompt):]
68
+ else:
69
+ generated = truth
70
+ starting_index = 0
71
+ all_input_ids = tokenizer(truth, add_special_tokens=False)["input_ids"]
72
if skip_special_tokens:
55
- all_input_ids = ([tokenizer.bos_token_id]
56
- if tokenizer.bos_token_id is not None else
57
- []) + all_input_ids + [tokenizer.eos_token_id]
73
+ if tokenizer.bos_token_id is not None:
74
+ all_input_ids = [tokenizer.bos_token_id] + all_input_ids
75
+ starting_index += 1
76
+ all_input_ids = all_input_ids + [tokenizer.eos_token_id]
77
78
decoded_text = _run_incremental_decode(
60
- tokenizer, all_input_ids, skip_special_tokens=skip_special_tokens)
79
+ tokenizer,
80
+ all_input_ids,
81
+ skip_special_tokens=skip_special_tokens,
82
+ starting_index=starting_index)
83
62
- assert decoded_text == truth
84
+ assert decoded_text == generated
85
+
86
+
87
+@pytest.fixture
88
+def detokenizer(tokenizer_name: str) -> Detokenizer:
89
+ init_kwargs = dict(
90
+ tokenizer_id=tokenizer_name,
91
+ enable_lora=False,
92
+ max_num_seqs=100,
93
+ max_input_length=None,
94
+ tokenizer_mode="auto",
95
+ trust_remote_code=False,
96
+ revision=None,
97
+ )
98
+
99
+ tokenizer_group = get_tokenizer_group(
100
+ None,
101
+ **init_kwargs,
102
+ )
103
+
104
+ return Detokenizer(tokenizer_group)
105
+
106
+
107
+@pytest.fixture(name="complete_sequence_token_ids")
108
+def create_complete_sequence_token_ids(complete_sequence: str,
109
+ tokenizer_name: str) -> List[int]:
110
+ tokenizer = AutoTokenizer.from_pretrained(tokenizer_name)
111
+ complete_sequence_token_ids = tokenizer(complete_sequence)["input_ids"]
112
+ return complete_sequence_token_ids
113
+
114
+
115
+def create_sequence(prompt_token_ids=None):
116
+ prompt_token_ids = prompt_token_ids or [1]
117
+ return Sequence(
118
+ seq_id=0,
119
+ prompt="<s>",
120
+ prompt_token_ids=prompt_token_ids,
121
+ block_size=16,
122
+ )
123
+
124
+
125
+def create_dummy_logprobs(
126
+ complete_sequence_token_ids: List[int]) -> List[Dict[int, Logprob]]:
127
+ return [{
128
+ token_id: Logprob(logprob=0.0),
129
+ token_id + 1: Logprob(logprob=0.1)
130
+ } for token_id in complete_sequence_token_ids]
131
+
132
+
133
+@pytest.mark.parametrize("complete_sequence", TRUTH)
134
+@pytest.mark.parametrize("tokenizer_name", TOKENIZERS)
135
+@pytest.mark.parametrize("skip_special_tokens", [True, False])
136
+def test_decode_sequence_logprobs(complete_sequence: str,
137
+ complete_sequence_token_ids: List[int],
138
+ detokenizer: Detokenizer,
139
+ skip_special_tokens: bool):
140
+ """Verify Detokenizer decodes logprobs correctly."""
141
+ sampling_params = SamplingParams(skip_special_tokens=skip_special_tokens,
142
+ logprobs=2)
143
+
144
+ # Run sequentially.
145
+ seq = create_sequence()
146
+ dummy_logprobs = create_dummy_logprobs(complete_sequence_token_ids)
147
+ sequential_logprobs_text_chosen_token = []
148
+ sequential_logprobs_text_other_token = []
149
+ for new_token, logprobs in zip(complete_sequence_token_ids,
150
+ dummy_logprobs):
151
+ seq.append_token_id(new_token, logprobs)
152
+ detokenizer.decode_sequence_inplace(seq, sampling_params)
153
+ sequential_logprobs_text_chosen_token.append(
154
+ seq.output_logprobs[-1][new_token].decoded_token)
155
+ sequential_logprobs_text_other_token.append(
156
+ seq.output_logprobs[-1][new_token + 1].decoded_token)
157
+ sequential_result = seq.output_text
158
+
159
+ assert sequential_result == "".join(sequential_logprobs_text_chosen_token)
160
+ assert sequential_result != "".join(sequential_logprobs_text_other_token)
161
+
162
+ if skip_special_tokens:
163
+ # Text for logprobs for the chosen token should be the same as the
164
+ # generated text. Note that this will only be true if we skip
165
+ # special tokens.
166
+ assert sequential_result == complete_sequence
167
+
168
+
169
+@pytest.mark.parametrize("complete_sequence", TRUTH)
170
+@pytest.mark.parametrize("tokenizer_name", TOKENIZERS)
171
+@pytest.mark.parametrize("skip_special_tokens", [True])
172
+def test_decode_prompt_logprobs(complete_sequence: str,
173
+ complete_sequence_token_ids: List[int],
174
+ detokenizer: Detokenizer,
175
+ skip_special_tokens: bool):
176
+ """Verify Detokenizer decodes prompt logprobs correctly."""
177
+ sampling_params = SamplingParams(skip_special_tokens=skip_special_tokens,
178
+ prompt_logprobs=1)
179
+
180
+ # Run sequentially.
181
+ seq = create_sequence(complete_sequence_token_ids)
182
+ seq_group = SequenceGroup(request_id="1",
183
+ seqs=[seq],
184
+ sampling_params=sampling_params,
185
+ arrival_time=0.0)
186
+ dummy_logprobs = create_dummy_logprobs(complete_sequence_token_ids)
187
+ detokenizer.decode_prompt_logprobs_inplace(seq_group, dummy_logprobs)
188
+ decoded_prompt_logprobs = dummy_logprobs
189
+
190
+ if skip_special_tokens:
191
+ # Text for logprobs for the chosen token should be the same as the
192
+ # prompt text. Note that this will only be true if we skip
193
+ # special tokens.
194
+ assert complete_sequence == "".join([
195
+ logprobs[token_id].decoded_token for token_id, logprobs in zip(
196
+ complete_sequence_token_ids, decoded_prompt_logprobs)
197
+ ])
198
+ assert complete_sequence != "".join([
199
+ logprobs[token_id + 1].decoded_token for token_id, logprobs in zip(
200
+ complete_sequence_token_ids, decoded_prompt_logprobs)
201
+ ])
vllm/engine/llm_engine.py
+9
-57
@@ -1,5 +1,5 @@
1
import time
2
-from typing import Dict, Iterable, List, Optional, Tuple, Type, Union
2
+from typing import Iterable, List, Optional, Tuple, Type, Union
3
4
from transformers import PreTrainedTokenizer
5
@@ -15,11 +15,11 @@ from vllm.engine.ray_utils import initialize_ray_cluster
15
from vllm.logger import init_logger
16
from vllm.outputs import RequestOutput
17
from vllm.sampling_params import SamplingParams
18
-from vllm.sequence import (Logprob, SamplerOutput, Sequence, SequenceGroup,
18
+from vllm.sequence import (SamplerOutput, Sequence, SequenceGroup,
19
SequenceGroupOutput, SequenceOutput, SequenceStatus)
20
-from vllm.transformers_utils.tokenizer import detokenize_incrementally
20
from vllm.transformers_utils.tokenizer_group import (BaseTokenizerGroup,
21
get_tokenizer_group)
22
+from vllm.transformers_utils.detokenizer import Detokenizer
23
from vllm.utils import Counter
24
25
logger = init_logger(__name__)
@@ -97,6 +97,7 @@ class LLMEngine:
97
self._verify_args()
98
99
self._init_tokenizer()
100
+ self.detokenizer = Detokenizer(self.tokenizer)
101
self.seq_counter = Counter()
102
103
self.model_executor = executor_class(model_config, cache_config,
@@ -153,7 +154,7 @@ class LLMEngine:
154
raise RuntimeError("LLMEngine should not be pickled!")
155
156
def get_tokenizer(self) -> "PreTrainedTokenizer":
156
- return self.tokenizer.get_lora_tokenizer()
157
+ return self.tokenizer.get_lora_tokenizer(None)
158
159
def get_tokenizer_for_seq(self,
160
sequence: Sequence) -> "PreTrainedTokenizer":
@@ -370,13 +371,8 @@ class LLMEngine:
371
# Process prompt logprobs
372
prompt_logprobs = outputs.prompt_logprobs
373
if prompt_logprobs is not None:
373
- # We can pick any sequence for the prompt.
374
- seq = next(iter(seq_group.seqs_dict.values()))
375
- all_token_ids = seq.get_token_ids()
376
- for i, prompt_logprobs_for_token in enumerate(prompt_logprobs):
377
- self._decode_logprobs(seq, seq_group.sampling_params,
378
- prompt_logprobs_for_token,
379
- all_token_ids[:i])
374
+ self.detokenizer.decode_prompt_logprobs_inplace(
375
+ seq_group, prompt_logprobs)
376
seq_group.prompt_logprobs = prompt_logprobs
377
378
# Process samples
@@ -420,7 +416,8 @@ class LLMEngine:
416
child_seqs.append((parent, parent))
417
418
for seq, _ in child_seqs:
423
- self._decode_sequence(seq, seq_group.sampling_params)
419
+ self.detokenizer.decode_sequence_inplace(seq,
420
+ seq_group.sampling_params)
421
self._check_stop(seq, seq_group.sampling_params)
422
423
# Non-beam search case
@@ -713,51 +710,6 @@ class LLMEngine:
710
time_e2e_requests=time_e2e_requests,
711
)
712
716
- def _decode_logprobs(self, seq: Sequence, prms: SamplingParams,
717
- logprobs: Dict[int, Logprob],
718
- all_input_ids: List[int]) -> None:
719
- if not logprobs:
720
- return
721
- for token_id, sample_logprob in logprobs.items():
722
- if (sample_logprob.decoded_token is None and token_id != -1):
723
- all_input_ids_with_logprob = all_input_ids[:-1] + [token_id]
724
- (_, new_text, prefix_offset,
725
- read_offset) = detokenize_incrementally(
726
- self.get_tokenizer_for_seq(seq),
727
- all_input_ids=all_input_ids_with_logprob,
728
- prev_tokens=seq.tokens,
729
- prefix_offset=seq.prefix_offset,
730
- read_offset=seq.read_offset,
731
- skip_special_tokens=prms.skip_special_tokens,
732
- spaces_between_special_tokens=prms.
733
- spaces_between_special_tokens,
734
- )
735
- sample_logprob.decoded_token = new_text
736
-
737
- def _decode_sequence(self, seq: Sequence, prms: SamplingParams) -> None:
738
- """Decodes the new token for a sequence."""
739
- all_input_ids = seq.get_token_ids()
740
- self._decode_logprobs(seq, prms, seq.output_logprobs[-1],
741
- all_input_ids)
742
-
743
- (new_tokens, new_output_text, prefix_offset,
744
- read_offset) = detokenize_incrementally(
745
- self.get_tokenizer_for_seq(seq),
746
- all_input_ids=all_input_ids,
747
- prev_tokens=seq.tokens,
748
- prefix_offset=seq.prefix_offset,
749
- read_offset=seq.read_offset,
750
- skip_special_tokens=prms.skip_special_tokens,
751
- spaces_between_special_tokens=prms.spaces_between_special_tokens,
752
- )
753
- if seq.tokens is None:
754
- seq.tokens = new_tokens
755
- else:
756
- seq.tokens.extend(new_tokens)
757
- seq.prefix_offset = prefix_offset
758
- seq.read_offset = read_offset
759
- seq.output_text += new_output_text
760
-
713
def _check_stop(self, seq: Sequence,
714
sampling_params: SamplingParams) -> None:
715
"""Stop the finished sequences."""
vllm/transformers_utils/detokenizer.py
new
+155
@@ -0,0 +1,155 @@
1
+from typing import List, Dict, Optional
2
+from transformers import PreTrainedTokenizer
3
+from vllm.sequence import Sequence, Logprob, SequenceGroup, SamplingParams
4
+from vllm.transformers_utils.tokenizer import (detokenize_incrementally,
5
+ convert_prompt_ids_to_tokens)
6
+from vllm.transformers_utils.tokenizer_group.base_tokenizer_group import (
7
+ BaseTokenizerGroup)
8
+
9
+# Used eg. for marking rejected tokens in spec decoding.
10
+INVALID_TOKEN_ID = -1
11
+
12
+
13
+class Detokenizer:
14
+ """Provides methods to decode the output of a model into text."""
15
+
16
+ def __init__(self, tokenizer_group: BaseTokenizerGroup):
17
+ self.tokenizer_group = tokenizer_group
18
+
19
+ def get_tokenizer_for_seq(self,
20
+ sequence: Sequence) -> "PreTrainedTokenizer":
21
+ """Returns the HF tokenizer to use for a given sequence."""
22
+ return self.tokenizer_group.get_lora_tokenizer(sequence.lora_request)
23
+
24
+ def decode_prompt_logprobs_inplace(
25
+ self, seq_group: SequenceGroup,
26
+ prompt_logprobs: List[Optional[Dict[int, Logprob]]]) -> None:
27
+ """Decodes the logprobs for the prompt of a sequence group.
28
+
29
+ Args:
30
+ seq_group: The sequence group to decode.
31
+ prompt_logprobs: The logprobs to decode.
32
+
33
+ Returns:
34
+ The prompt logprobs with the decoded tokens.
35
+ """
36
+ prms = seq_group.sampling_params
37
+ # We can pick any sequence for the prompt.
38
+ seq = next(iter(seq_group.seqs_dict.values()))
39
+ # Only prompt, without the generated token.
40
+ all_token_ids = seq.get_token_ids()
41
+ prompt_token_ids = all_token_ids[:-1]
42
+ tokenizer = self.get_tokenizer_for_seq(seq)
43
+ prefix_offset = 0
44
+ read_offset = 0
45
+ next_iter_prefix_offset = 0
46
+ next_iter_read_offset = 0
47
+ next_iter_tokens = []
48
+ prev_tokens = None
49
+
50
+ for token_position, prompt_logprobs_for_token in enumerate(
51
+ prompt_logprobs):
52
+ if not prompt_logprobs_for_token:
53
+ continue
54
+ for token_id, sample_logprob in prompt_logprobs_for_token.items():
55
+ if (sample_logprob.decoded_token is None
56
+ and token_id != INVALID_TOKEN_ID):
57
+ prompt_token_ids_with_token = (
58
+ prompt_token_ids[:token_position] + [token_id])
59
+ (new_tokens, new_text, new_prefix_offset,
60
+ new_read_offset) = detokenize_incrementally(
61
+ tokenizer=tokenizer,
62
+ all_input_ids=prompt_token_ids_with_token,
63
+ prev_tokens=prev_tokens,
64
+ prefix_offset=prefix_offset,
65
+ read_offset=read_offset,
66
+ skip_special_tokens=prms.skip_special_tokens,
67
+ spaces_between_special_tokens=prms.
68
+ spaces_between_special_tokens,
69
+ )
70
+
71
+ sample_logprob.decoded_token = new_text
72
+
73
+ # Use the offsets & prev tokens corresponding to
74
+ # real tokens to ensure detokenization is consistent
75
+ # actual with prompt.
76
+ if token_id == all_token_ids[token_position]:
77
+ next_iter_prefix_offset = new_prefix_offset
78
+ next_iter_read_offset = new_read_offset
79
+ next_iter_tokens = new_tokens
80
+
81
+ # Advance to the next token position.
82
+ prefix_offset = next_iter_prefix_offset
83
+ read_offset = next_iter_read_offset
84
+ if prev_tokens is None:
85
+ prev_tokens = next_iter_tokens
86
+ else:
87
+ prev_tokens.extend(next_iter_tokens)
88
+
89
+ def decode_sequence_inplace(self, seq: Sequence,
90
+ prms: SamplingParams) -> None:
91
+ """Decodes the new token for a sequence. In-place operation.
92
+
93
+ Args:
94
+ seq: The sequence to decode.
95
+ prms: The sampling parameters used to generate the sequence.
96
+ """
97
+ all_input_ids = seq.get_token_ids()
98
+ token_id_generated_this_iteration = all_input_ids[-1]
99
+ tokenizer = self.get_tokenizer_for_seq(seq)
100
+
101
+ # Convert prompt token IDs to tokens if necessary.
102
+ # Do it here so that we don't have to repeat this
103
+ # computation for each logprob.
104
+ if seq.tokens is None:
105
+ (seq.tokens, seq.prefix_offset,
106
+ seq.read_offset) = convert_prompt_ids_to_tokens(
107
+ tokenizer=tokenizer,
108
+ prompt_ids=all_input_ids[:-1],
109
+ skip_special_tokens=prms.skip_special_tokens,
110
+ )
111
+
112
+ (new_tokens, new_decoded_token_text, prefix_offset,
113
+ read_offset) = detokenize_incrementally(
114
+ tokenizer=tokenizer,
115
+ all_input_ids=all_input_ids,
116
+ prev_tokens=seq.tokens,
117
+ prefix_offset=seq.prefix_offset,
118
+ read_offset=seq.read_offset,
119
+ skip_special_tokens=prms.skip_special_tokens,
120
+ spaces_between_special_tokens=prms.spaces_between_special_tokens,
121
+ )
122
+
123
+ # Decode logprobs
124
+ logprobs = seq.output_logprobs[-1]
125
+ if logprobs:
126
+ previous_tokens = all_input_ids[:-1]
127
+ for token_id, sample_logprob in logprobs.items():
128
+ # If the token was generated this iteration,
129
+ # use the provided text.
130
+ if token_id == token_id_generated_this_iteration:
131
+ sample_logprob.decoded_token = new_decoded_token_text
132
+ continue
133
+
134
+ if (sample_logprob.decoded_token is None
135
+ and token_id != INVALID_TOKEN_ID):
136
+ all_input_ids_with_logprob = previous_tokens + [token_id]
137
+ (_, new_text, _, _) = detokenize_incrementally(
138
+ tokenizer=tokenizer,
139
+ all_input_ids=all_input_ids_with_logprob,
140
+ prev_tokens=seq.tokens,
141
+ prefix_offset=seq.prefix_offset,
142
+ read_offset=seq.read_offset,
143
+ skip_special_tokens=prms.skip_special_tokens,
144
+ spaces_between_special_tokens=prms.
145
+ spaces_between_special_tokens,
146
+ )
147
+ sample_logprob.decoded_token = new_text
148
+
149
+ if seq.tokens is None:
150
+ seq.tokens = new_tokens
151
+ else:
152
+ seq.tokens.extend(new_tokens)
153
+ seq.prefix_offset = prefix_offset
154
+ seq.read_offset = read_offset
155
+ seq.output_text += new_decoded_token_text
vllm/transformers_utils/tokenizer.py
+70
-20
@@ -158,6 +158,34 @@ def _convert_tokens_to_string_with_added_encoders(
158
return "".join(sub_texts)
159
160
161
+# 5 is an arbitrary value that should work for all
162
+# tokenizers (bigger = more conservative).
163
+INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET = 5
164
+
165
+
166
+def convert_prompt_ids_to_tokens(
167
+ tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
168
+ prompt_ids: List[int],
169
+ skip_special_tokens: bool = False,
170
+) -> Tuple[List[str], int, int]:
171
+ """Converts the prompt ids to tokens and returns the tokens and offsets
172
+ for incremental detokenization.
173
+
174
+ Note that not all tokens are converted to strings. Only the tokens that
175
+ are necessary for incremental detokenization are converted to strings.
176
+ """
177
+ # Offset a little more in case we have special tokens.
178
+ prefix_offset = max(
179
+ len(prompt_ids) - INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET - 2, 0)
180
+ # We do not need to convert the whole prompt to tokens.
181
+ new_tokens = tokenizer.convert_ids_to_tokens(
182
+ prompt_ids[prefix_offset:], skip_special_tokens=skip_special_tokens)
183
+ prefix_offset = max(
184
+ len(new_tokens) - INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET, 0)
185
+ read_offset = len(new_tokens)
186
+ return new_tokens, prefix_offset, read_offset
187
+
188
+
189
# Based on
190
# https://github.com/huggingface/text-generation-inference/blob/v0.9.4/server/text_generation_server/models/model.py#L62C9-L62C15
191
# under Apache 2.0 license
@@ -165,31 +193,53 @@ def detokenize_incrementally(
193
tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
194
all_input_ids: List[int],
195
prev_tokens: Optional[List[str]],
168
- prefix_offset: int = 0,
169
- read_offset: int = 0,
196
+ prefix_offset: int,
197
+ read_offset: int,
198
skip_special_tokens: bool = False,
199
spaces_between_special_tokens: bool = True,
200
) -> Tuple[List[str], str, int, int]:
201
+ """Detokenizes the input ids incrementally and returns the new tokens
202
+ and the new text.
203
+
204
+ If `prev_tokens` is None, this function will convert the input ids to
205
+ tokens and return the tokens and the new text. Otherwise, it will return the
206
+ new tokens and the new text.
207
+
208
+ This function will also return the new prefix offset and the new read
209
+ offset to be used in the next iteration.
210
+
211
+ The offsets are necessary to defeat cleanup algorithms in the decode which
212
+ decide to add a space or not depending on the surrounding ids.
213
+
214
+ Args:
215
+ tokenizer: The tokenizer to use.
216
+ all_input_ids: The input ids. The last id is the new token id.
217
+ prev_tokens: The previous tokens. If None, this function will convert
218
+ the input ids to tokens and return the tokens and the new text.
219
+ prefix_offset: The prefix offset.
220
+ read_offset: The read offset.
221
+ skip_special_tokens: Whether to skip special tokens.
222
+ spaces_between_special_tokens: Whether to add spaces between special
223
+ tokens.
224
+ """
225
new_token_id = all_input_ids[-1]
226
# This is the first iteration for this sequence
175
- if prev_tokens is None:
176
- new_tokens = tokenizer.convert_ids_to_tokens(
177
- all_input_ids, skip_special_tokens=skip_special_tokens)
178
- output_tokens = new_tokens
179
- # 5 is an arbitrary value that should work for all
180
- # tokenizers (bigger = more conservative).
181
- # Subtract 1 extra to account for the generated token.
182
- prefix_offset = max(len(output_tokens) - 6, 0)
183
- # If the first new token is a special token, we can't skip 1 extra token
184
- if skip_special_tokens and new_token_id in tokenizer.all_special_ids:
185
- read_offset = max(len(output_tokens), 0)
186
- else:
187
- read_offset = max(len(output_tokens) - 1, 0)
188
- else:
189
- # Put new_token_id in a list so skip_special_tokens is respected
190
- new_tokens = tokenizer.convert_ids_to_tokens(
191
- [new_token_id], skip_special_tokens=skip_special_tokens)
192
- output_tokens = prev_tokens + new_tokens
227
+ is_first_iter = prev_tokens is None
228
+ if is_first_iter:
229
+ (prev_tokens, prefix_offset,
230
+ read_offset) = convert_prompt_ids_to_tokens(
231
+ tokenizer,
232
+ all_input_ids[:-1],
233
+ skip_special_tokens=skip_special_tokens)
234
+
235
+ # Put new_token_id in a list so skip_special_tokens is respected
236
+ new_tokens = tokenizer.convert_ids_to_tokens(
237
+ [new_token_id], skip_special_tokens=skip_special_tokens)
238
+ output_tokens = prev_tokens + new_tokens
239
+
240
+ # If this is the first iteration, return all tokens.
241
+ if is_first_iter:
242
+ new_tokens = output_tokens
243
244
# The prefix text is necessary only to defeat cleanup algorithms in
245
# the decode which decide to add a space or not depending on the