A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 14:45 UTC
[V1][Perf] Faster incremental detokenization (#15137)
Signed-off-by: Nick Hill <nhill@redhat.com>
Nick Hill committed
Apr 17, 2025 at 07:45 UTC
05fcd1b4308aa2dca9a1b24c540b57a94a7ba124
7 files changed
+316
-144
requirements/common.txt
+1
-1
@@ -8,7 +8,7 @@ blake3
8
py-cpuinfo
9
transformers >= 4.51.1
10
huggingface-hub[hf_xet] >= 0.30.0 # Required for Xet downloads.
11
-tokenizers >= 0.19.1 # Required for Llama 3.
11
+tokenizers >= 0.21.1 # Required for fast incremental detokenization.
12
protobuf # Required by LlamaTokenizer.
13
fastapi[standard] >= 0.115.0 # Required by FastAPI's form models in the OpenAI API server's audio transcriptions endpoint.
14
aiohttp
requirements/test.in
+1
@@ -35,6 +35,7 @@ opencv-python-headless >= 4.11.0 # required for video test
35
datamodel_code_generator # required for minicpm3 test
36
lm-eval[api]==0.4.8 # required for model evaluation test
37
transformers==4.51.1
38
+tokenizers==0.21.1
39
huggingface-hub[hf_xet]>=0.30.0 # Required for Xet downloads.
40
# quantization
41
bitsandbytes>=0.45.3
requirements/test.txt
+4
-2
@@ -624,8 +624,10 @@ tiktoken==0.7.0
624
# mistral-common
625
timm==1.0.11
626
# via -r requirements/test.in
627
-tokenizers==0.21.0
628
- # via transformers
627
+tokenizers==0.21.1
628
+ # via
629
+ # -r requirements/test.in
630
+ # transformers
631
torch==2.6.0
632
# via
633
# -r requirements/test.in
tests/lora/test_llama_tp.py
+1
@@ -47,6 +47,7 @@ def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> list[str]:
47
]
48
sampling_params = vllm.SamplingParams(temperature=0,
49
max_tokens=256,
50
+ skip_special_tokens=False,
51
stop=["[/assistant]"])
52
outputs = llm.generate(
53
prompts,
tests/tokenization/test_detokenize.py
+137
-55
@@ -4,14 +4,22 @@ from collections.abc import Generator
4
from typing import Any, Optional
5
6
import pytest
7
-from transformers import AutoTokenizer
7
+from transformers import (AutoTokenizer, PreTrainedTokenizer,
8
+ PreTrainedTokenizerFast)
9
10
from vllm.inputs import token_inputs
11
from vllm.sequence import Logprob, SamplingParams, Sequence, SequenceGroup
11
-from vllm.transformers_utils.detokenizer import (Detokenizer,
12
- detokenize_incrementally)
12
+from vllm.transformers_utils.detokenizer import Detokenizer
13
from vllm.transformers_utils.tokenizer_group import get_tokenizer_group
14
from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer
15
+from vllm.v1.engine import EngineCoreRequest
16
+from vllm.v1.engine.detokenizer import (FastIncrementalDetokenizer,
17
+ IncrementalDetokenizer,
18
+ SlowIncrementalDetokenizer)
19
+
20
+SPECIAL_TOKS_TRUTH = [
21
+ "Some text with adjacent special tokens <|padding|><|padding|><fim_prefix><fim_middle><fim_suffix>other text<fim_pad>", # noqa
22
+]
23
24
TRUTH = [
25
"Hello here, this is a simple test",
@@ -22,7 +30,8 @@ TRUTH = [
30
# incomplete UTF-8 characters
31
# see https://github.com/vllm-project/vllm/pull/9625
32
"ပုံပြင်လေးပြောပြပါ်",
25
-]
33
+] + SPECIAL_TOKS_TRUTH
34
+
35
TOKENIZERS = [
36
"facebook/opt-125m",
37
"gpt2",
@@ -38,26 +47,37 @@ TOKENIZERS = [
47
]
48
49
41
-def _run_incremental_decode(tokenizer, all_input_ids,
42
- skip_special_tokens: bool, starting_index: int):
43
- decoded_text = ""
44
- offset = 0
45
- token_offset = 0
46
- prev_tokens = None
47
- for i in range(starting_index, len(all_input_ids)):
48
- new_tokens, text, offset, token_offset = detokenize_incrementally(
49
- tokenizer,
50
- all_input_ids[:i + 1],
51
- prev_tokens,
52
- offset,
53
- token_offset,
54
- skip_special_tokens=skip_special_tokens)
55
- decoded_text += text
56
- if prev_tokens is None:
57
- prev_tokens = new_tokens
58
- else:
59
- prev_tokens += new_tokens
60
- return decoded_text
50
+def _run_incremental_decode(tokenizer,
51
+ all_input_ids,
52
+ skip_special_tokens: bool,
53
+ starting_index: int,
54
+ spaces_between_special_tokens: bool = True,
55
+ fast: Optional[bool] = None):
56
+
57
+ prompt_token_ids = all_input_ids[:starting_index]
58
+
59
+ params = SamplingParams(
60
+ skip_special_tokens=skip_special_tokens,
61
+ spaces_between_special_tokens=spaces_between_special_tokens,
62
+ )
63
+ request = EngineCoreRequest("", "", prompt_token_ids, None, None, None,
64
+ params, None, 0.0, None)
65
+
66
+ if fast is None:
67
+ detokenizer = IncrementalDetokenizer.from_new_request(
68
+ tokenizer, request)
69
+ elif fast:
70
+ detokenizer = FastIncrementalDetokenizer(tokenizer, request)
71
+ else:
72
+ detokenizer = SlowIncrementalDetokenizer(tokenizer, request)
73
+
74
+ output_text = ""
75
+ for i, token_id in enumerate(all_input_ids[starting_index:]):
76
+ detokenizer.update([token_id], False)
77
+ finished = i == len(all_input_ids) - 1
78
+ output_text += detokenizer.get_next_output_text(finished, delta=True)
79
+
80
+ return output_text, detokenizer.output_token_ids
81
82
83
@pytest.fixture
@@ -85,11 +105,13 @@ def test_mistral_edge_case(tokenizer, truth):
105
starting_index = 0
106
all_input_ids = tokenizer(truth, add_special_tokens=False).input_ids
107
88
- decoded_text = _run_incremental_decode(tokenizer,
89
- all_input_ids,
90
- skip_special_tokens=True,
91
- starting_index=starting_index)
108
+ decoded_text, out_ids = _run_incremental_decode(
109
+ tokenizer,
110
+ all_input_ids,
111
+ skip_special_tokens=True,
112
+ starting_index=starting_index)
113
assert decoded_text == truth
114
+ assert out_ids == all_input_ids[starting_index:]
115
116
117
@pytest.fixture
@@ -106,40 +128,86 @@ def skip_special_tokens(request, tokenizer_name) -> Generator[bool, Any, None]:
128
@pytest.mark.parametrize("with_prompt", [True, False])
129
@pytest.mark.parametrize("tokenizer_name", TOKENIZERS)
130
@pytest.mark.parametrize("skip_special_tokens", (True, False), indirect=True)
109
-def test_decode_streaming(tokenizer, truth, with_prompt, skip_special_tokens):
131
+@pytest.mark.parametrize("spaces_between_special_tokens", (True, False))
132
+@pytest.mark.parametrize("fast", (True, False))
133
+def test_decode_streaming(tokenizer, truth, with_prompt, skip_special_tokens,
134
+ spaces_between_special_tokens, fast):
135
+ if fast and not isinstance(tokenizer, PreTrainedTokenizerFast):
136
+ pytest.skip()
137
+
138
+ if skip_special_tokens and not spaces_between_special_tokens:
139
+ pytest.skip()
140
+
141
+ if not fast and isinstance(tokenizer, PreTrainedTokenizerFast):
142
+ # Fix up inconsistency in fast/slow tokenizer behaviour.
143
+ tokenizer.add_special_tokens({
144
+ "additional_special_tokens": [
145
+ at for at in
146
+ tokenizer._tokenizer.get_added_tokens_decoder().values()
147
+ if at.special
148
+ ]
149
+ })
150
+
151
+ extra_decode_args = {} if not isinstance(tokenizer, PreTrainedTokenizer) \
152
+ else {"spaces_between_special_tokens": spaces_between_special_tokens}
153
+
154
+ truth_tokens = tokenizer(truth, add_special_tokens=False).input_ids
155
+ if tokenizer.bos_token_id is not None:
156
+ truth_tokens.insert(0, tokenizer.bos_token_id)
157
+ truth_tokens.append(tokenizer.eos_token_id)
158
+
159
+ new_truth = tokenizer.decode(truth_tokens,
160
+ skip_special_tokens=skip_special_tokens,
161
+ **extra_decode_args)
162
+
163
if with_prompt:
111
- truth_tokens = tokenizer(truth, add_special_tokens=False).input_ids
112
- prompt_input_ids = truth_tokens[:len(truth) // 2]
113
- generated_input_ids = truth_tokens[len(truth) // 2:]
164
+ num_prompt_tokens = len(
165
+ tokenizer(truth[:len(truth) // 2],
166
+ add_special_tokens=False).input_ids)
167
+ if tokenizer.bos_token_id is not None:
168
+ num_prompt_tokens += 1
169
+
170
+ prompt_input_ids = truth_tokens[:num_prompt_tokens]
171
+ generated_input_ids = truth_tokens[num_prompt_tokens:]
172
all_input_ids = prompt_input_ids + generated_input_ids
173
starting_index = len(prompt_input_ids)
174
prompt = tokenizer.decode(prompt_input_ids,
117
- skip_special_tokens=skip_special_tokens)
118
- generated = truth[len(prompt):]
175
+ skip_special_tokens=skip_special_tokens,
176
+ **extra_decode_args)
177
+
178
+ generated = new_truth[len(prompt):]
179
else:
120
- generated = truth
180
+ generated = new_truth
181
starting_index = 0
122
- all_input_ids = tokenizer(truth, add_special_tokens=False).input_ids
123
- if skip_special_tokens:
124
- if tokenizer.bos_token_id is not None:
125
- all_input_ids = [tokenizer.bos_token_id] + all_input_ids
126
- starting_index += 1
127
- all_input_ids = all_input_ids + [tokenizer.eos_token_id]
182
+ all_input_ids = truth_tokens
183
129
- decoded_text = _run_incremental_decode(
184
+ decoded_text, out_ids = _run_incremental_decode(
185
tokenizer,
186
all_input_ids,
187
skip_special_tokens=skip_special_tokens,
133
- starting_index=starting_index)
188
+ starting_index=starting_index,
189
+ spaces_between_special_tokens=spaces_between_special_tokens,
190
+ fast=fast)
191
192
assert decoded_text == generated
193
+ assert out_ids == all_input_ids[starting_index:]
194
137
- decoded_text = _run_incremental_decode(
195
+
196
+@pytest.mark.parametrize("tokenizer_name", TOKENIZERS)
197
+@pytest.mark.parametrize("fast", (True, False))
198
+def test_oov_decode(tokenizer, fast):
199
+ if fast and not isinstance(tokenizer, PreTrainedTokenizerFast):
200
+ pytest.skip()
201
+
202
+ decoded_text, out_ids = _run_incremental_decode(
203
tokenizer, [len(tokenizer)],
139
- skip_special_tokens=skip_special_tokens,
140
- starting_index=starting_index)
204
+ skip_special_tokens=True,
205
+ starting_index=0,
206
+ spaces_between_special_tokens=True,
207
+ fast=fast)
208
209
assert decoded_text == ''
210
+ assert out_ids == [len(tokenizer)]
211
212
213
@pytest.fixture
@@ -165,15 +233,14 @@ def detokenizer(tokenizer_name: str) -> Detokenizer:
233
@pytest.fixture(name="complete_sequence_token_ids")
234
def create_complete_sequence_token_ids(complete_sequence: str,
235
tokenizer) -> list[int]:
168
- complete_sequence_token_ids = tokenizer(complete_sequence).input_ids
169
- return complete_sequence_token_ids
236
+ return tokenizer(complete_sequence, add_special_tokens=False).input_ids
237
238
239
def create_sequence(prompt_token_ids=None):
173
- prompt_token_ids = prompt_token_ids or [1]
240
+ prompt_token_ids = prompt_token_ids or []
241
return Sequence(
242
seq_id=0,
176
- inputs=token_inputs(prompt_token_ids, prompt="<s>"),
243
+ inputs=token_inputs(prompt_token_ids),
244
block_size=16,
245
)
246
@@ -224,7 +291,7 @@ def test_decode_sequence_logprobs(complete_sequence: str,
291
assert sequential_result == "".join(sequential_logprobs_text_chosen_token)
292
assert sequential_result != "".join(sequential_logprobs_text_other_token)
293
227
- if skip_special_tokens:
294
+ if not skip_special_tokens:
295
# Text for logprobs for the chosen token should be the same as the
296
# generated text. Note that this will only be true if we skip
297
# special tokens.
@@ -233,10 +300,23 @@ def test_decode_sequence_logprobs(complete_sequence: str,
300
301
@pytest.mark.parametrize("complete_sequence", TRUTH)
302
@pytest.mark.parametrize("tokenizer_name", TOKENIZERS)
236
-def test_decode_prompt_logprobs(complete_sequence_token_ids: list[int],
303
+def test_decode_prompt_logprobs(complete_sequence: str,
304
+ complete_sequence_token_ids: list[int],
305
detokenizer: Detokenizer):
306
+
307
+ # We want to use skip_special_tokens=False here but Mistral tokenizers
308
+ # don't support that.
309
+ if complete_sequence not in SPECIAL_TOKS_TRUTH:
310
+ skip_special_tokens = True
311
+ elif not isinstance(detokenizer.tokenizer_group.get_lora_tokenizer(None),
312
+ MistralTokenizer):
313
+ skip_special_tokens = False
314
+ else:
315
+ pytest.skip("MistralTokenizers don't support "
316
+ "skip_special_tokens=False")
317
+ return
318
"""Verify Detokenizer decodes prompt logprobs correctly."""
239
- sampling_params = SamplingParams(skip_special_tokens=True,
319
+ sampling_params = SamplingParams(skip_special_tokens=skip_special_tokens,
320
prompt_logprobs=1)
321
322
# Run sequentially.
@@ -256,8 +336,10 @@ def test_decode_prompt_logprobs(complete_sequence_token_ids: list[int],
336
# decoded_prompt_logprobs doesn't contain the first token.
337
token_ids = complete_sequence_token_ids
338
tokenizer = detokenizer.get_tokenizer_for_seq(seq)
259
- text_full = tokenizer.decode(token_ids, skip_special_tokens=True)
260
- text_first = tokenizer.decode(token_ids[0], skip_special_tokens=True)
339
+ text_full = tokenizer.decode(token_ids,
340
+ skip_special_tokens=skip_special_tokens)
341
+ text_first = tokenizer.decode(token_ids[0],
342
+ skip_special_tokens=skip_special_tokens)
343
text = text_full[len(text_first):]
344
345
# Text for logprobs for the chosen token should be the same as the
vllm/entrypoints/openai/tool_parsers/mistral_tool_parser.py
+9
@@ -70,6 +70,15 @@ class MistralToolParser(ToolParser):
70
"Mistral Tool Parser could not locate the tool call token in "
71
"the tokenizer!")
72
73
+ def adjust_request(
74
+ self, request: ChatCompletionRequest) -> ChatCompletionRequest:
75
+ if request.tools and request.tool_choice != 'none':
76
+ # do not skip special tokens because mistral uses the special
77
+ # tokens to indicate the start and end of the tool calls
78
+ # information.
79
+ request.skip_special_tokens = False
80
+ return request
81
+
82
def extract_tool_calls(
83
self,
84
model_output: str,
vllm/v1/engine/detokenizer.py
+163
-86
@@ -1,8 +1,11 @@
1
# SPDX-License-Identifier: Apache-2.0
2
-
3
-from dataclasses import dataclass, field
2
+from abc import ABC, abstractmethod
3
from typing import Optional
4
5
+from tokenizers import Tokenizer
6
+from tokenizers.decoders import DecodeStream
7
+from transformers import PreTrainedTokenizerFast
8
+
9
from vllm.engine.output_processor.stop_checker import StopChecker
10
from vllm.logger import init_logger
11
from vllm.transformers_utils.detokenizer_utils import (
@@ -12,39 +15,22 @@ from vllm.v1.engine import EngineCoreRequest
15
logger = init_logger(__name__)
16
17
15
-@dataclass
18
class IncrementalDetokenizer:
19
18
- # Generation data
19
- token_ids: list[int]
20
- output_text: str = ""
21
- tokens: list[str] = field(default_factory=list)
22
- prompt_len: int = 0
23
-
24
- # Stop strings
25
- stop: list[str] = field(default_factory=list)
26
- include_stop_str_in_output: bool = False
27
-
28
- # Metadata for incremental detokenization
29
- prefix_offset: int = 0
30
- read_offset: int = 0
31
-
32
- # Parameters for detokenization
33
- skip_special_tokens: bool = True
34
- spaces_between_special_tokens: bool = True
35
-
36
- # Tokenizer for this request,
37
- # None if detokenization is disabled.
38
- tokenizer: Optional[AnyTokenizer] = None
39
-
40
- # Accounting for stop string buffering
41
- stop_buffer_length: int = 0
42
- _last_output_text_offset: int = 0
20
+ def __init__(self):
21
+ self.token_ids: list[int] = []
22
23
@property
24
def output_token_ids(self) -> list[int]:
46
- return self.token_ids if not self.prompt_len else (
47
- self.token_ids[self.prompt_len:])
25
+ return self.token_ids
26
+
27
+ def update(self, new_token_ids: list[int],
28
+ stop_terminated: bool) -> Optional[str]:
29
+ self.token_ids.extend(new_token_ids)
30
+ return None
31
+
32
+ def get_next_output_text(self, finished: bool, delta: bool) -> str:
33
+ return ""
34
35
@classmethod
36
def from_new_request(
@@ -54,39 +40,37 @@ class IncrementalDetokenizer:
40
) -> "IncrementalDetokenizer":
41
42
if tokenizer is None:
57
- return cls(token_ids=[])
43
+ # No tokenizer => skipping detokenization.
44
+ return IncrementalDetokenizer()
45
+
46
+ if isinstance(tokenizer, PreTrainedTokenizerFast):
47
+ # Fast tokenizer => use tokenizers library DecodeStream.
48
+ return FastIncrementalDetokenizer(tokenizer, request)
49
+
50
+ # Fall back to slow python-based incremental detokenization.
51
+ return SlowIncrementalDetokenizer(tokenizer, request)
52
+
53
+
54
+class BaseIncrementalDetokenizer(IncrementalDetokenizer, ABC):
55
+
56
+ def __init__(self, request: EngineCoreRequest):
57
+ super().__init__()
58
59
- tokens, prefix_offset, read_offset = convert_prompt_ids_to_tokens(
60
- tokenizer=tokenizer,
61
- prompt_ids=request.prompt_token_ids,
62
- skip_special_tokens=request.sampling_params.skip_special_tokens,
63
- )
59
+ # Stop strings
60
+ params = request.sampling_params
61
+ self.stop = stop = params.stop
62
+ self.include_stop_str_in_output = params.include_stop_str_in_output
63
65
- stops = request.sampling_params.stop
64
# Number of chars to hold back when stop strings are to be excluded
65
# from streamed output.
68
- if stops and not request.sampling_params.include_stop_str_in_output:
69
- stop_buffer_length = max(len(s) for s in stops) - 1
66
+ if stop and not self.include_stop_str_in_output:
67
+ self.stop_buffer_length = max(len(s) for s in stop) - 1
68
else:
71
- stop_buffer_length = 0
72
-
73
- return cls(
74
- tokens=tokens,
75
- # Detokenizer mutates this list, so need a unique copy.
76
- # NOTE(Nick): could we take ownership of it though?
77
- token_ids=request.prompt_token_ids.copy(),
78
- stop=stops,
79
- include_stop_str_in_output=request.sampling_params.
80
- include_stop_str_in_output,
81
- prefix_offset=prefix_offset,
82
- read_offset=read_offset,
83
- skip_special_tokens=request.sampling_params.skip_special_tokens,
84
- spaces_between_special_tokens=request.sampling_params.
85
- spaces_between_special_tokens,
86
- prompt_len=len(request.prompt_token_ids),
87
- tokenizer=tokenizer,
88
- stop_buffer_length=stop_buffer_length,
89
- )
69
+ self.stop_buffer_length = 0
70
+ self._last_output_text_offset: int = 0
71
+
72
+ # Generation data
73
+ self.output_text = ""
74
75
def update(self, new_token_ids: list[int],
76
stop_terminated: bool) -> Optional[str]:
@@ -98,11 +82,7 @@ class IncrementalDetokenizer:
82
Return matched stop string or None.
83
"""
84
if not new_token_ids:
101
- # Skip detokenization if no new token ids
102
- return None
103
- if self.tokenizer is None:
104
- # Skip detokenization if no tokenizer
105
- self.token_ids.extend(new_token_ids)
85
+ # Skip detokenization if no new token ids.
86
return None
87
88
if stop_terminated and not self.include_stop_str_in_output:
@@ -116,34 +96,16 @@ class IncrementalDetokenizer:
96
# 1) Detokenize the new token ids incrementally.
97
# TODO(woosuk): This method becomes very inefficient when the number of
98
# new_token_ids is more than 1. We need to optimize this.
119
- decoded_text = ""
99
+ offset_before = len(self.output_text)
100
for new_token_id in new_token_ids:
101
self.token_ids.append(new_token_id)
122
- (new_tokens, new_decoded_token_text, prefix_offset,
123
- read_offset) = detokenize_incrementally(
124
- tokenizer=self.tokenizer,
125
- all_input_ids=self.token_ids,
126
- prev_tokens=self.tokens,
127
- prefix_offset=self.prefix_offset,
128
- read_offset=self.read_offset,
129
- skip_special_tokens=self.skip_special_tokens,
130
- spaces_between_special_tokens=self.
131
- spaces_between_special_tokens,
132
- )
133
-
134
- self.tokens.extend(new_tokens)
135
- self.prefix_offset = prefix_offset
136
- self.read_offset = read_offset
137
-
138
- decoded_text += new_decoded_token_text
139
-
140
- self.output_text += decoded_text
102
+ self.output_text += self.decode_next(new_token_id)
103
104
if stop_terminated:
105
if skipped_stop_token_id is not None:
144
- # Cleanup after skipping detokenization
106
+ # Cleanup after skipping detokenization.
107
self.token_ids.append(skipped_stop_token_id)
146
- # Stop token triggered; skip stop string check
108
+ # Stop token triggered; skip stop string check.
109
return None
110
111
# 2) Evaluate stop strings.
@@ -151,7 +113,7 @@ class IncrementalDetokenizer:
113
if self.stop:
114
stop = StopChecker.check_stop_strings(
115
output_text=self.output_text,
154
- new_char_count=len(decoded_text),
116
+ new_char_count=len(self.output_text) - offset_before,
117
stop=self.stop,
118
include_in_output=self.include_stop_str_in_output,
119
)
@@ -162,6 +124,10 @@ class IncrementalDetokenizer:
124
125
return stop_string
126
127
+ @abstractmethod
128
+ def decode_next(self, next_token_id: int) -> str:
129
+ raise NotImplementedError
130
+
131
def get_next_output_text(self, finished: bool, delta: bool) -> str:
132
"""If delta is True, only new text since the last call to
133
this method is returned"""
@@ -177,3 +143,114 @@ class IncrementalDetokenizer:
143
self._last_output_text_offset = length
144
return self.output_text[last_offset:length]
145
return ""
146
+
147
+
148
+class FastIncrementalDetokenizer(BaseIncrementalDetokenizer):
149
+
150
+ def __init__(self, tokenizer: PreTrainedTokenizerFast,
151
+ request: EngineCoreRequest):
152
+ super().__init__(request)
153
+
154
+ sampling_params = request.sampling_params
155
+ self.stream = DecodeStream(
156
+ skip_special_tokens=sampling_params.skip_special_tokens)
157
+
158
+ self.tokenizer: Tokenizer = tokenizer._tokenizer
159
+
160
+ # Find a safe place to start.
161
+ prompt_suffix = request.prompt_token_ids
162
+ prompt_len = len(prompt_suffix)
163
+ if prompt_len > 4:
164
+ for i in range(4, max(prompt_len + 1, 32)):
165
+ suffix = request.prompt_token_ids[-i:]
166
+ if '�' not in self.tokenizer.decode(suffix):
167
+ prompt_suffix = suffix
168
+ break
169
+
170
+ # Prime the stream.
171
+ for tid in prompt_suffix:
172
+ self.stream.step(self.tokenizer, tid)
173
+
174
+ self.spaces_between_special_tokens = (
175
+ sampling_params.skip_special_tokens
176
+ or sampling_params.spaces_between_special_tokens)
177
+
178
+ if not self.spaces_between_special_tokens:
179
+ # Store dict of added token ids so that we can suppress
180
+ # the spaces between them.
181
+ if (added_token_ids := getattr(self.tokenizer, "added_token_ids",
182
+ None)) is None:
183
+ self.tokenizer.added_token_ids = added_token_ids = {
184
+ tid: tok.content
185
+ for tid, tok in
186
+ self.tokenizer.get_added_tokens_decoder().items()
187
+ }
188
+
189
+ if added_token_ids:
190
+ self.last_special = False
191
+ self.added_token_ids = added_token_ids
192
+ else:
193
+ # No added tokens.
194
+ self.spaces_between_special_tokens = True
195
+
196
+ def decode_next(self, next_token_id: int) -> str:
197
+ token = self.stream.step(self.tokenizer, next_token_id)
198
+
199
+ if not self.spaces_between_special_tokens:
200
+ special_token = self.added_token_ids.get(next_token_id)
201
+ is_special = special_token is not None
202
+ if is_special and self.last_special:
203
+ # Return raw token string without any prefixed spaces.
204
+ token = special_token
205
+ self.last_special = is_special
206
+
207
+ return token or ""
208
+
209
+
210
+class SlowIncrementalDetokenizer(BaseIncrementalDetokenizer):
211
+
212
+ def __init__(self, tokenizer: AnyTokenizer, request: EngineCoreRequest):
213
+ super().__init__(request)
214
+
215
+ self.tokenizer = tokenizer
216
+
217
+ # Metadata for incremental detokenization.
218
+ self.tokens, self.prefix_offset, self.read_offset = (
219
+ convert_prompt_ids_to_tokens(
220
+ tokenizer=tokenizer,
221
+ prompt_ids=request.prompt_token_ids,
222
+ skip_special_tokens=request.sampling_params.
223
+ skip_special_tokens,
224
+ ))
225
+
226
+ self.token_ids.extend(request.prompt_token_ids)
227
+ self.prompt_len = len(request.prompt_token_ids)
228
+
229
+ params = request.sampling_params
230
+ self.skip_special_tokens = params.skip_special_tokens
231
+ self.spaces_between_special_tokens = (
232
+ params.spaces_between_special_tokens)
233
+
234
+ @property
235
+ def output_token_ids(self) -> list[int]:
236
+ return self.token_ids if not self.prompt_len else (
237
+ self.token_ids[self.prompt_len:])
238
+
239
+ def decode_next(self, next_token_id: int) -> str:
240
+ new_tokens, decoded_text, prefix_offset, read_offset = (
241
+ detokenize_incrementally(
242
+ tokenizer=self.tokenizer,
243
+ all_input_ids=self.token_ids,
244
+ prev_tokens=self.tokens,
245
+ prefix_offset=self.prefix_offset,
246
+ read_offset=self.read_offset,
247
+ skip_special_tokens=self.skip_special_tokens,
248
+ spaces_between_special_tokens=self.
249
+ spaces_between_special_tokens,
250
+ ))
251
+
252
+ self.tokens.extend(new_tokens)
253
+ self.prefix_offset = prefix_offset
254
+ self.read_offset = read_offset
255
+
256
+ return decoded_text