@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / bfdb1ba5c3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[Core] Improve detokenization performance for prefill (#3469)

Co-authored-by: MeloYang <meloyang05@gmail.com>

Antoni Baum committed Mar 22, 2024 at 13:44 UTC bfdb1ba5c3fb14387c69acb1f5067102d8028e56
4 files changed +385 -89
tests/tokenization/test_detokenize.py
+151 -12
@@ -1,13 +1,17 @@
1 import pytest
2
3 from transformers import AutoTokenizer
4 +from typing import List, Dict
5
6 +from vllm.sequence import Sequence, Logprob, SamplingParams, SequenceGroup
7 +from vllm.transformers_utils.tokenizer_group import get_tokenizer_group
8 from vllm.transformers_utils.tokenizer import detokenize_incrementally
9 +from vllm.transformers_utils.detokenizer import Detokenizer
10
11 TRUTH = [
8 - "Hello here, this is a simple test", # noqa: E501
9 - "vLLM is a high-throughput and memory-efficient inference and serving engine for LLMs. It is designed to be used in production environments, where inference and serving", # noqa: E501
10 - "我很感谢你的热情" # noqa: E501
12 + "Hello here, this is a simple test",
13 + "vLLM is a high-throughput and memory-efficient inference and serving engine for LLMs. It is designed to be used in production environments, where inference and serving", # noqa
14 + "我很感谢你的热情"
15 ]
16 TOKENIZERS = [
17 "facebook/opt-125m",
@@ -24,12 +28,12 @@ TOKENIZERS = [
28
29
30 def _run_incremental_decode(tokenizer, all_input_ids,
27 - skip_special_tokens: bool):
31 + skip_special_tokens: bool, starting_index: int):
32 decoded_text = ""
33 offset = 0
34 token_offset = 0
35 prev_tokens = None
32 - for i in range(len(all_input_ids)):
36 + for i in range(starting_index, len(all_input_ids)):
37 new_tokens, text, offset, token_offset = detokenize_incrementally(
38 tokenizer,
39 all_input_ids[:i + 1],
@@ -46,17 +50,152 @@ def _run_incremental_decode(tokenizer, all_input_ids,
50
51
52 @pytest.mark.parametrize("truth", TRUTH)
53 +@pytest.mark.parametrize("with_prompt", [True, False])
54 @pytest.mark.parametrize("tokenizer_id", TOKENIZERS)
55 @pytest.mark.parametrize("skip_special_tokens", (True, False))
51 -def test_decode_streaming(tokenizer_id, truth, skip_special_tokens):
56 +def test_decode_streaming(tokenizer_id, truth, with_prompt,
57 + skip_special_tokens):
58 tokenizer = AutoTokenizer.from_pretrained(tokenizer_id)
53 - all_input_ids = tokenizer(truth, add_special_tokens=False)["input_ids"]
59 + if with_prompt:
60 + truth_tokens = tokenizer(truth, add_special_tokens=False)["input_ids"]
61 + prompt_input_ids = truth_tokens[:len(truth) // 2]
62 + generated_input_ids = truth_tokens[len(truth) // 2:]
63 + all_input_ids = prompt_input_ids + generated_input_ids
64 + starting_index = len(prompt_input_ids)
65 + prompt = tokenizer.decode(prompt_input_ids,
66 + skip_special_tokens=skip_special_tokens)
67 + generated = truth[len(prompt):]
68 + else:
69 + generated = truth
70 + starting_index = 0
71 + all_input_ids = tokenizer(truth, add_special_tokens=False)["input_ids"]
72 if skip_special_tokens:
55 - all_input_ids = ([tokenizer.bos_token_id]
56 - if tokenizer.bos_token_id is not None else
57 - []) + all_input_ids + [tokenizer.eos_token_id]
73 + if tokenizer.bos_token_id is not None:
74 + all_input_ids = [tokenizer.bos_token_id] + all_input_ids
75 + starting_index += 1
76 + all_input_ids = all_input_ids + [tokenizer.eos_token_id]
77
78 decoded_text = _run_incremental_decode(
60 - tokenizer, all_input_ids, skip_special_tokens=skip_special_tokens)
79 + tokenizer,
80 + all_input_ids,
81 + skip_special_tokens=skip_special_tokens,
82 + starting_index=starting_index)
83
62 - assert decoded_text == truth
84 + assert decoded_text == generated
85 +
86 +
87 +@pytest.fixture
88 +def detokenizer(tokenizer_name: str) -> Detokenizer:
89 + init_kwargs = dict(
90 + tokenizer_id=tokenizer_name,
91 + enable_lora=False,
92 + max_num_seqs=100,
93 + max_input_length=None,
94 + tokenizer_mode="auto",
95 + trust_remote_code=False,
96 + revision=None,
97 + )
98 +
99 + tokenizer_group = get_tokenizer_group(
100 + None,
101 + **init_kwargs,
102 + )
103 +
104 + return Detokenizer(tokenizer_group)
105 +
106 +
107 +@pytest.fixture(name="complete_sequence_token_ids")
108 +def create_complete_sequence_token_ids(complete_sequence: str,
109 + tokenizer_name: str) -> List[int]:
110 + tokenizer = AutoTokenizer.from_pretrained(tokenizer_name)
111 + complete_sequence_token_ids = tokenizer(complete_sequence)["input_ids"]
112 + return complete_sequence_token_ids
113 +
114 +
115 +def create_sequence(prompt_token_ids=None):
116 + prompt_token_ids = prompt_token_ids or [1]
117 + return Sequence(
118 + seq_id=0,
119 + prompt="<s>",
120 + prompt_token_ids=prompt_token_ids,
121 + block_size=16,
122 + )
123 +
124 +
125 +def create_dummy_logprobs(
126 + complete_sequence_token_ids: List[int]) -> List[Dict[int, Logprob]]:
127 + return [{
128 + token_id: Logprob(logprob=0.0),
129 + token_id + 1: Logprob(logprob=0.1)
130 + } for token_id in complete_sequence_token_ids]
131 +
132 +
133 +@pytest.mark.parametrize("complete_sequence", TRUTH)
134 +@pytest.mark.parametrize("tokenizer_name", TOKENIZERS)
135 +@pytest.mark.parametrize("skip_special_tokens", [True, False])
136 +def test_decode_sequence_logprobs(complete_sequence: str,
137 + complete_sequence_token_ids: List[int],
138 + detokenizer: Detokenizer,
139 + skip_special_tokens: bool):
140 + """Verify Detokenizer decodes logprobs correctly."""
141 + sampling_params = SamplingParams(skip_special_tokens=skip_special_tokens,
142 + logprobs=2)
143 +
144 + # Run sequentially.
145 + seq = create_sequence()
146 + dummy_logprobs = create_dummy_logprobs(complete_sequence_token_ids)
147 + sequential_logprobs_text_chosen_token = []
148 + sequential_logprobs_text_other_token = []
149 + for new_token, logprobs in zip(complete_sequence_token_ids,
150 + dummy_logprobs):
151 + seq.append_token_id(new_token, logprobs)
152 + detokenizer.decode_sequence_inplace(seq, sampling_params)
153 + sequential_logprobs_text_chosen_token.append(
154 + seq.output_logprobs[-1][new_token].decoded_token)
155 + sequential_logprobs_text_other_token.append(
156 + seq.output_logprobs[-1][new_token + 1].decoded_token)
157 + sequential_result = seq.output_text
158 +
159 + assert sequential_result == "".join(sequential_logprobs_text_chosen_token)
160 + assert sequential_result != "".join(sequential_logprobs_text_other_token)
161 +
162 + if skip_special_tokens:
163 + # Text for logprobs for the chosen token should be the same as the
164 + # generated text. Note that this will only be true if we skip
165 + # special tokens.
166 + assert sequential_result == complete_sequence
167 +
168 +
169 +@pytest.mark.parametrize("complete_sequence", TRUTH)
170 +@pytest.mark.parametrize("tokenizer_name", TOKENIZERS)
171 +@pytest.mark.parametrize("skip_special_tokens", [True])
172 +def test_decode_prompt_logprobs(complete_sequence: str,
173 + complete_sequence_token_ids: List[int],
174 + detokenizer: Detokenizer,
175 + skip_special_tokens: bool):
176 + """Verify Detokenizer decodes prompt logprobs correctly."""
177 + sampling_params = SamplingParams(skip_special_tokens=skip_special_tokens,
178 + prompt_logprobs=1)
179 +
180 + # Run sequentially.
181 + seq = create_sequence(complete_sequence_token_ids)
182 + seq_group = SequenceGroup(request_id="1",
183 + seqs=[seq],
184 + sampling_params=sampling_params,
185 + arrival_time=0.0)
186 + dummy_logprobs = create_dummy_logprobs(complete_sequence_token_ids)
187 + detokenizer.decode_prompt_logprobs_inplace(seq_group, dummy_logprobs)
188 + decoded_prompt_logprobs = dummy_logprobs
189 +
190 + if skip_special_tokens:
191 + # Text for logprobs for the chosen token should be the same as the
192 + # prompt text. Note that this will only be true if we skip
193 + # special tokens.
194 + assert complete_sequence == "".join([
195 + logprobs[token_id].decoded_token for token_id, logprobs in zip(
196 + complete_sequence_token_ids, decoded_prompt_logprobs)
197 + ])
198 + assert complete_sequence != "".join([
199 + logprobs[token_id + 1].decoded_token for token_id, logprobs in zip(
200 + complete_sequence_token_ids, decoded_prompt_logprobs)
201 + ])
vllm/engine/llm_engine.py
+9 -57
@@ -1,5 +1,5 @@
1 import time
2 -from typing import Dict, Iterable, List, Optional, Tuple, Type, Union
2 +from typing import Iterable, List, Optional, Tuple, Type, Union
3
4 from transformers import PreTrainedTokenizer
5
@@ -15,11 +15,11 @@ from vllm.engine.ray_utils import initialize_ray_cluster
15 from vllm.logger import init_logger
16 from vllm.outputs import RequestOutput
17 from vllm.sampling_params import SamplingParams
18 -from vllm.sequence import (Logprob, SamplerOutput, Sequence, SequenceGroup,
18 +from vllm.sequence import (SamplerOutput, Sequence, SequenceGroup,
19 SequenceGroupOutput, SequenceOutput, SequenceStatus)
20 -from vllm.transformers_utils.tokenizer import detokenize_incrementally
20 from vllm.transformers_utils.tokenizer_group import (BaseTokenizerGroup,
21 get_tokenizer_group)
22 +from vllm.transformers_utils.detokenizer import Detokenizer
23 from vllm.utils import Counter
24
25 logger = init_logger(__name__)
@@ -97,6 +97,7 @@ class LLMEngine:
97 self._verify_args()
98
99 self._init_tokenizer()
100 + self.detokenizer = Detokenizer(self.tokenizer)
101 self.seq_counter = Counter()
102
103 self.model_executor = executor_class(model_config, cache_config,
@@ -153,7 +154,7 @@ class LLMEngine:
154 raise RuntimeError("LLMEngine should not be pickled!")
155
156 def get_tokenizer(self) -> "PreTrainedTokenizer":
156 - return self.tokenizer.get_lora_tokenizer()
157 + return self.tokenizer.get_lora_tokenizer(None)
158
159 def get_tokenizer_for_seq(self,
160 sequence: Sequence) -> "PreTrainedTokenizer":
@@ -370,13 +371,8 @@ class LLMEngine:
371 # Process prompt logprobs
372 prompt_logprobs = outputs.prompt_logprobs
373 if prompt_logprobs is not None:
373 - # We can pick any sequence for the prompt.
374 - seq = next(iter(seq_group.seqs_dict.values()))
375 - all_token_ids = seq.get_token_ids()
376 - for i, prompt_logprobs_for_token in enumerate(prompt_logprobs):
377 - self._decode_logprobs(seq, seq_group.sampling_params,
378 - prompt_logprobs_for_token,
379 - all_token_ids[:i])
374 + self.detokenizer.decode_prompt_logprobs_inplace(
375 + seq_group, prompt_logprobs)
376 seq_group.prompt_logprobs = prompt_logprobs
377
378 # Process samples
@@ -420,7 +416,8 @@ class LLMEngine:
416 child_seqs.append((parent, parent))
417
418 for seq, _ in child_seqs:
423 - self._decode_sequence(seq, seq_group.sampling_params)
419 + self.detokenizer.decode_sequence_inplace(seq,
420 + seq_group.sampling_params)
421 self._check_stop(seq, seq_group.sampling_params)
422
423 # Non-beam search case
@@ -713,51 +710,6 @@ class LLMEngine:
710 time_e2e_requests=time_e2e_requests,
711 )
712
716 - def _decode_logprobs(self, seq: Sequence, prms: SamplingParams,
717 - logprobs: Dict[int, Logprob],
718 - all_input_ids: List[int]) -> None:
719 - if not logprobs:
720 - return
721 - for token_id, sample_logprob in logprobs.items():
722 - if (sample_logprob.decoded_token is None and token_id != -1):
723 - all_input_ids_with_logprob = all_input_ids[:-1] + [token_id]
724 - (_, new_text, prefix_offset,
725 - read_offset) = detokenize_incrementally(
726 - self.get_tokenizer_for_seq(seq),
727 - all_input_ids=all_input_ids_with_logprob,
728 - prev_tokens=seq.tokens,
729 - prefix_offset=seq.prefix_offset,
730 - read_offset=seq.read_offset,
731 - skip_special_tokens=prms.skip_special_tokens,
732 - spaces_between_special_tokens=prms.
733 - spaces_between_special_tokens,
734 - )
735 - sample_logprob.decoded_token = new_text
736 -
737 - def _decode_sequence(self, seq: Sequence, prms: SamplingParams) -> None:
738 - """Decodes the new token for a sequence."""
739 - all_input_ids = seq.get_token_ids()
740 - self._decode_logprobs(seq, prms, seq.output_logprobs[-1],
741 - all_input_ids)
742 -
743 - (new_tokens, new_output_text, prefix_offset,
744 - read_offset) = detokenize_incrementally(
745 - self.get_tokenizer_for_seq(seq),
746 - all_input_ids=all_input_ids,
747 - prev_tokens=seq.tokens,
748 - prefix_offset=seq.prefix_offset,
749 - read_offset=seq.read_offset,
750 - skip_special_tokens=prms.skip_special_tokens,
751 - spaces_between_special_tokens=prms.spaces_between_special_tokens,
752 - )
753 - if seq.tokens is None:
754 - seq.tokens = new_tokens
755 - else:
756 - seq.tokens.extend(new_tokens)
757 - seq.prefix_offset = prefix_offset
758 - seq.read_offset = read_offset
759 - seq.output_text += new_output_text
760 -
713 def _check_stop(self, seq: Sequence,
714 sampling_params: SamplingParams) -> None:
715 """Stop the finished sequences."""
vllm/transformers_utils/detokenizer.py new
+155
@@ -0,0 +1,155 @@
1 +from typing import List, Dict, Optional
2 +from transformers import PreTrainedTokenizer
3 +from vllm.sequence import Sequence, Logprob, SequenceGroup, SamplingParams
4 +from vllm.transformers_utils.tokenizer import (detokenize_incrementally,
5 + convert_prompt_ids_to_tokens)
6 +from vllm.transformers_utils.tokenizer_group.base_tokenizer_group import (
7 + BaseTokenizerGroup)
8 +
9 +# Used eg. for marking rejected tokens in spec decoding.
10 +INVALID_TOKEN_ID = -1
11 +
12 +
13 +class Detokenizer:
14 + """Provides methods to decode the output of a model into text."""
15 +
16 + def __init__(self, tokenizer_group: BaseTokenizerGroup):
17 + self.tokenizer_group = tokenizer_group
18 +
19 + def get_tokenizer_for_seq(self,
20 + sequence: Sequence) -> "PreTrainedTokenizer":
21 + """Returns the HF tokenizer to use for a given sequence."""
22 + return self.tokenizer_group.get_lora_tokenizer(sequence.lora_request)
23 +
24 + def decode_prompt_logprobs_inplace(
25 + self, seq_group: SequenceGroup,
26 + prompt_logprobs: List[Optional[Dict[int, Logprob]]]) -> None:
27 + """Decodes the logprobs for the prompt of a sequence group.
28 +
29 + Args:
30 + seq_group: The sequence group to decode.
31 + prompt_logprobs: The logprobs to decode.
32 +
33 + Returns:
34 + The prompt logprobs with the decoded tokens.
35 + """
36 + prms = seq_group.sampling_params
37 + # We can pick any sequence for the prompt.
38 + seq = next(iter(seq_group.seqs_dict.values()))
39 + # Only prompt, without the generated token.
40 + all_token_ids = seq.get_token_ids()
41 + prompt_token_ids = all_token_ids[:-1]
42 + tokenizer = self.get_tokenizer_for_seq(seq)
43 + prefix_offset = 0
44 + read_offset = 0
45 + next_iter_prefix_offset = 0
46 + next_iter_read_offset = 0
47 + next_iter_tokens = []
48 + prev_tokens = None
49 +
50 + for token_position, prompt_logprobs_for_token in enumerate(
51 + prompt_logprobs):
52 + if not prompt_logprobs_for_token:
53 + continue
54 + for token_id, sample_logprob in prompt_logprobs_for_token.items():
55 + if (sample_logprob.decoded_token is None
56 + and token_id != INVALID_TOKEN_ID):
57 + prompt_token_ids_with_token = (
58 + prompt_token_ids[:token_position] + [token_id])
59 + (new_tokens, new_text, new_prefix_offset,
60 + new_read_offset) = detokenize_incrementally(
61 + tokenizer=tokenizer,
62 + all_input_ids=prompt_token_ids_with_token,
63 + prev_tokens=prev_tokens,
64 + prefix_offset=prefix_offset,
65 + read_offset=read_offset,
66 + skip_special_tokens=prms.skip_special_tokens,
67 + spaces_between_special_tokens=prms.
68 + spaces_between_special_tokens,
69 + )
70 +
71 + sample_logprob.decoded_token = new_text
72 +
73 + # Use the offsets & prev tokens corresponding to
74 + # real tokens to ensure detokenization is consistent
75 + # actual with prompt.
76 + if token_id == all_token_ids[token_position]:
77 + next_iter_prefix_offset = new_prefix_offset
78 + next_iter_read_offset = new_read_offset
79 + next_iter_tokens = new_tokens
80 +
81 + # Advance to the next token position.
82 + prefix_offset = next_iter_prefix_offset
83 + read_offset = next_iter_read_offset
84 + if prev_tokens is None:
85 + prev_tokens = next_iter_tokens
86 + else:
87 + prev_tokens.extend(next_iter_tokens)
88 +
89 + def decode_sequence_inplace(self, seq: Sequence,
90 + prms: SamplingParams) -> None:
91 + """Decodes the new token for a sequence. In-place operation.
92 +
93 + Args:
94 + seq: The sequence to decode.
95 + prms: The sampling parameters used to generate the sequence.
96 + """
97 + all_input_ids = seq.get_token_ids()
98 + token_id_generated_this_iteration = all_input_ids[-1]
99 + tokenizer = self.get_tokenizer_for_seq(seq)
100 +
101 + # Convert prompt token IDs to tokens if necessary.
102 + # Do it here so that we don't have to repeat this
103 + # computation for each logprob.
104 + if seq.tokens is None:
105 + (seq.tokens, seq.prefix_offset,
106 + seq.read_offset) = convert_prompt_ids_to_tokens(
107 + tokenizer=tokenizer,
108 + prompt_ids=all_input_ids[:-1],
109 + skip_special_tokens=prms.skip_special_tokens,
110 + )
111 +
112 + (new_tokens, new_decoded_token_text, prefix_offset,
113 + read_offset) = detokenize_incrementally(
114 + tokenizer=tokenizer,
115 + all_input_ids=all_input_ids,
116 + prev_tokens=seq.tokens,
117 + prefix_offset=seq.prefix_offset,
118 + read_offset=seq.read_offset,
119 + skip_special_tokens=prms.skip_special_tokens,
120 + spaces_between_special_tokens=prms.spaces_between_special_tokens,
121 + )
122 +
123 + # Decode logprobs
124 + logprobs = seq.output_logprobs[-1]
125 + if logprobs:
126 + previous_tokens = all_input_ids[:-1]
127 + for token_id, sample_logprob in logprobs.items():
128 + # If the token was generated this iteration,
129 + # use the provided text.
130 + if token_id == token_id_generated_this_iteration:
131 + sample_logprob.decoded_token = new_decoded_token_text
132 + continue
133 +
134 + if (sample_logprob.decoded_token is None
135 + and token_id != INVALID_TOKEN_ID):
136 + all_input_ids_with_logprob = previous_tokens + [token_id]
137 + (_, new_text, _, _) = detokenize_incrementally(
138 + tokenizer=tokenizer,
139 + all_input_ids=all_input_ids_with_logprob,
140 + prev_tokens=seq.tokens,
141 + prefix_offset=seq.prefix_offset,
142 + read_offset=seq.read_offset,
143 + skip_special_tokens=prms.skip_special_tokens,
144 + spaces_between_special_tokens=prms.
145 + spaces_between_special_tokens,
146 + )
147 + sample_logprob.decoded_token = new_text
148 +
149 + if seq.tokens is None:
150 + seq.tokens = new_tokens
151 + else:
152 + seq.tokens.extend(new_tokens)
153 + seq.prefix_offset = prefix_offset
154 + seq.read_offset = read_offset
155 + seq.output_text += new_decoded_token_text
vllm/transformers_utils/tokenizer.py
+70 -20
@@ -158,6 +158,34 @@ def _convert_tokens_to_string_with_added_encoders(
158 return "".join(sub_texts)
159
160
161 +# 5 is an arbitrary value that should work for all
162 +# tokenizers (bigger = more conservative).
163 +INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET = 5
164 +
165 +
166 +def convert_prompt_ids_to_tokens(
167 + tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
168 + prompt_ids: List[int],
169 + skip_special_tokens: bool = False,
170 +) -> Tuple[List[str], int, int]:
171 + """Converts the prompt ids to tokens and returns the tokens and offsets
172 + for incremental detokenization.
173 +
174 + Note that not all tokens are converted to strings. Only the tokens that
175 + are necessary for incremental detokenization are converted to strings.
176 + """
177 + # Offset a little more in case we have special tokens.
178 + prefix_offset = max(
179 + len(prompt_ids) - INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET - 2, 0)
180 + # We do not need to convert the whole prompt to tokens.
181 + new_tokens = tokenizer.convert_ids_to_tokens(
182 + prompt_ids[prefix_offset:], skip_special_tokens=skip_special_tokens)
183 + prefix_offset = max(
184 + len(new_tokens) - INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET, 0)
185 + read_offset = len(new_tokens)
186 + return new_tokens, prefix_offset, read_offset
187 +
188 +
189 # Based on
190 # https://github.com/huggingface/text-generation-inference/blob/v0.9.4/server/text_generation_server/models/model.py#L62C9-L62C15
191 # under Apache 2.0 license
@@ -165,31 +193,53 @@ def detokenize_incrementally(
193 tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
194 all_input_ids: List[int],
195 prev_tokens: Optional[List[str]],
168 - prefix_offset: int = 0,
169 - read_offset: int = 0,
196 + prefix_offset: int,
197 + read_offset: int,
198 skip_special_tokens: bool = False,
199 spaces_between_special_tokens: bool = True,
200 ) -> Tuple[List[str], str, int, int]:
201 + """Detokenizes the input ids incrementally and returns the new tokens
202 + and the new text.
203 +
204 + If `prev_tokens` is None, this function will convert the input ids to
205 + tokens and return the tokens and the new text. Otherwise, it will return the
206 + new tokens and the new text.
207 +
208 + This function will also return the new prefix offset and the new read
209 + offset to be used in the next iteration.
210 +
211 + The offsets are necessary to defeat cleanup algorithms in the decode which
212 + decide to add a space or not depending on the surrounding ids.
213 +
214 + Args:
215 + tokenizer: The tokenizer to use.
216 + all_input_ids: The input ids. The last id is the new token id.
217 + prev_tokens: The previous tokens. If None, this function will convert
218 + the input ids to tokens and return the tokens and the new text.
219 + prefix_offset: The prefix offset.
220 + read_offset: The read offset.
221 + skip_special_tokens: Whether to skip special tokens.
222 + spaces_between_special_tokens: Whether to add spaces between special
223 + tokens.
224 + """
225 new_token_id = all_input_ids[-1]
226 # This is the first iteration for this sequence
175 - if prev_tokens is None:
176 - new_tokens = tokenizer.convert_ids_to_tokens(
177 - all_input_ids, skip_special_tokens=skip_special_tokens)
178 - output_tokens = new_tokens
179 - # 5 is an arbitrary value that should work for all
180 - # tokenizers (bigger = more conservative).
181 - # Subtract 1 extra to account for the generated token.
182 - prefix_offset = max(len(output_tokens) - 6, 0)
183 - # If the first new token is a special token, we can't skip 1 extra token
184 - if skip_special_tokens and new_token_id in tokenizer.all_special_ids:
185 - read_offset = max(len(output_tokens), 0)
186 - else:
187 - read_offset = max(len(output_tokens) - 1, 0)
188 - else:
189 - # Put new_token_id in a list so skip_special_tokens is respected
190 - new_tokens = tokenizer.convert_ids_to_tokens(
191 - [new_token_id], skip_special_tokens=skip_special_tokens)
192 - output_tokens = prev_tokens + new_tokens
227 + is_first_iter = prev_tokens is None
228 + if is_first_iter:
229 + (prev_tokens, prefix_offset,
230 + read_offset) = convert_prompt_ids_to_tokens(
231 + tokenizer,
232 + all_input_ids[:-1],
233 + skip_special_tokens=skip_special_tokens)
234 +
235 + # Put new_token_id in a list so skip_special_tokens is respected
236 + new_tokens = tokenizer.convert_ids_to_tokens(
237 + [new_token_id], skip_special_tokens=skip_special_tokens)
238 + output_tokens = prev_tokens + new_tokens
239 +
240 + # If this is the first iteration, return all tokens.
241 + if is_first_iter:
242 + new_tokens = output_tokens
243
244 # The prefix text is necessary only to defeat cleanup algorithms in
245 # the decode which decide to add a space or not depending on the