[Bugfix] Fix logprobs token-string collision from SentencePiece space… (#48674)
Signed-off-by: Allen Shen <aoshen@inferact.ai> Co-authored-by: mvanhorn <mvanhorn@users.noreply.github.com> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
aoshen02 committed
Jul 20, 2026 at 12:17 UTC
dcfebf93f4eccf30f71872283331eee757915daf
2 files changed
+194
-8
tests/tokenizers_/test_detokenize.py
+120
@@ -8,6 +8,7 @@ import pytest
8
from transformers import AutoTokenizer, PythonBackend, TokenizersBackend
9
10
from vllm.sampling_params import SamplingParams
11
+from vllm.tokenizers.detokenizer_utils import convert_ids_list_to_tokens
12
from vllm.tokenizers.mistral import MistralTokenizer
13
from vllm.v1.engine import EngineCoreRequest
14
from vllm.v1.engine.detokenizer import (
@@ -239,3 +240,122 @@ def test_oov_decode(tokenizer, fast):
240
241
assert decoded_text == ""
242
assert out_ids == [len(tokenizer)]
243
+
244
+
245
+# ---------- convert_ids_list_to_tokens collision tests ----------
246
+
247
+
248
+class _MockBackend:
249
+ """Fake backend_tokenizer that exposes pre_tokenizer config."""
250
+
251
+ def __init__(self, pre_tokenizer_type, replacement=None):
252
+ import json
253
+
254
+ pre: dict = {"type": pre_tokenizer_type}
255
+ if replacement is not None:
256
+ pre["replacement"] = replacement
257
+ self._config = json.dumps({"pre_tokenizer": pre})
258
+
259
+ def to_str(self):
260
+ return self._config
261
+
262
+
263
+class _MockTokenizer:
264
+ """Minimal tokenizer mock for testing convert_ids_list_to_tokens."""
265
+
266
+ def __init__(
267
+ self,
268
+ raw_tokens: dict[int, str],
269
+ decoded_tokens: dict[int, str],
270
+ pre_tokenizer_type: str = "Metaspace",
271
+ replacement: str | None = "▁",
272
+ ):
273
+ self._raw = raw_tokens
274
+ self._decoded = decoded_tokens
275
+ self.backend_tokenizer = _MockBackend(pre_tokenizer_type, replacement)
276
+
277
+ def convert_ids_to_tokens(
278
+ self, ids: list[int], skip_special_tokens: bool = False
279
+ ) -> list[str]:
280
+ return [self._raw[tid] for tid in ids]
281
+
282
+ def decode(self, ids: list[int], skip_special_tokens: bool = False) -> str:
283
+ return "".join(self._decoded[tid] for tid in ids)
284
+
285
+
286
+def test_sentencepiece_leading_space_preserved():
287
+ """▁true and true must produce distinct strings."""
288
+ tok = _MockTokenizer(
289
+ raw_tokens={0: "▁true", 1: "true", 2: "▁false", 3: "false"},
290
+ decoded_tokens={0: "true", 1: "true", 2: "false", 3: "false"},
291
+ )
292
+ result = convert_ids_list_to_tokens(tok, [0, 1, 2, 3])
293
+ assert result == [" true", "true", " false", "false"]
294
+
295
+ # No dict collision when used as top_logprobs keys
296
+ logprobs = dict(zip(result, [-0.1, -0.2, -0.3, -0.4]))
297
+ assert len(logprobs) == 4
298
+
299
+
300
+def test_whitespace_run_tokens_stay_distinct():
301
+ """▁, ▁▁, ▁▁▁ must produce different-length space strings."""
302
+ tok = _MockTokenizer(
303
+ raw_tokens={0: "▁", 1: "▁▁", 2: "▁▁▁"},
304
+ decoded_tokens={0: "", 1: " ", 2: " "},
305
+ )
306
+ result = convert_ids_list_to_tokens(tok, [0, 1, 2])
307
+ assert result == [" ", " ", " "]
308
+
309
+
310
+def test_bpe_leading_space_already_preserved():
311
+ """GPT-2 BPE: Ġtrue already decodes to ' true', no fix needed."""
312
+ tok = _MockTokenizer(
313
+ raw_tokens={0: "Ġtrue", 1: "true"},
314
+ decoded_tokens={0: " true", 1: "true"},
315
+ pre_tokenizer_type="ByteLevel",
316
+ replacement=None,
317
+ )
318
+ result = convert_ids_list_to_tokens(tok, [0, 1])
319
+ assert result == [" true", "true"]
320
+
321
+
322
+def test_logprobs_count_stable_across_k():
323
+ """logprobs=4 and logprobs=10 must return 4 and 10 entries."""
324
+ tok = _MockTokenizer(
325
+ raw_tokens={
326
+ 0: "▁true",
327
+ 1: "a",
328
+ 2: "b",
329
+ 3: "c",
330
+ 4: "true",
331
+ 5: "d",
332
+ 6: "e",
333
+ 7: "f",
334
+ 8: "g",
335
+ 9: "h",
336
+ },
337
+ decoded_tokens={
338
+ 0: "true",
339
+ 1: "a",
340
+ 2: "b",
341
+ 3: "c",
342
+ 4: "true",
343
+ 5: "d",
344
+ 6: "e",
345
+ 7: "f",
346
+ 8: "g",
347
+ 9: "h",
348
+ },
349
+ )
350
+ ids = list(range(10))
351
+ lps = [-0.1 * (i + 1) for i in range(10)]
352
+
353
+ tokens4 = convert_ids_list_to_tokens(tok, ids[:4])
354
+ top4 = dict(zip(tokens4, lps[:4]))
355
+
356
+ tokens10 = convert_ids_list_to_tokens(tok, ids)
357
+ top10 = dict(zip(tokens10, lps))
358
+
359
+ assert len(top4) == 4
360
+ assert len(top10) == 10
361
+ assert top4[" true"] == top10[" true"]
vllm/tokenizers/detokenizer_utils.py
+74
-8
@@ -1,6 +1,9 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
+from __future__ import annotations
5
+
6
+import json
7
8
from vllm.tokenizers import TokenizerLike
9
@@ -56,6 +59,63 @@ def _convert_tokens_to_string_with_added_encoders(
59
INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET = 5
60
61
62
+_CACHED_MARKER_KEY = "_vllm_space_marker_cache"
63
+_NOT_CACHED = "__not_computed__"
64
+
65
+
66
+def _get_leading_space_marker(tokenizer: TokenizerLike) -> str | None:
67
+ """Read the space marker from the tokenizer's pre_tokenizer config.
68
+
69
+ Only Metaspace pre_tokenizers (used by SentencePiece-based models like
70
+ Llama, Mistral, T5) have a replacement character whose leading instance
71
+ gets stripped by decode(). ByteLevel (GPT-2), BertPreTokenizer (BERT),
72
+ and others do not have this issue.
73
+
74
+ Returns the marker character, or None if decode() is safe for single
75
+ tokens.
76
+ """
77
+ cached = getattr(tokenizer, _CACHED_MARKER_KEY, _NOT_CACHED)
78
+ if cached is not _NOT_CACHED:
79
+ return cached # type: ignore[return-value]
80
+
81
+ backend = getattr(tokenizer, "backend_tokenizer", None)
82
+ if backend is None:
83
+ result = None
84
+ else:
85
+ result = None
86
+ try:
87
+ config = json.loads(backend.to_str())
88
+ except Exception:
89
+ pass
90
+ else:
91
+ pre = config.get("pre_tokenizer") or {}
92
+ pre_type = pre.get("type")
93
+ if pre_type == "Metaspace":
94
+ result = pre.get("replacement", "▁")
95
+ elif pre_type == "Sequence":
96
+ for sub in pre.get("pretokenizers", []):
97
+ if sub.get("type") == "Metaspace":
98
+ result = sub.get("replacement", "▁")
99
+ break
100
+
101
+ setattr(tokenizer, _CACHED_MARKER_KEY, result)
102
+ return result
103
+
104
+
105
+def _restore_leading_spaces(raw_token: str, token_str: str, marker: str) -> str:
106
+ """Restore leading spaces that decode() stripped from a raw vocab piece."""
107
+ num_markers = 0
108
+ for ch in raw_token:
109
+ if ch != marker:
110
+ break
111
+ num_markers += 1
112
+ if num_markers == 0:
113
+ return token_str
114
+ existing = len(token_str) - len(token_str.lstrip(" "))
115
+ missing = num_markers - existing
116
+ return " " * missing + token_str if missing > 0 else token_str
117
+
118
+
119
def convert_prompt_ids_to_tokens(
120
tokenizer: TokenizerLike,
121
prompt_ids: list[int],
@@ -86,6 +146,10 @@ def convert_ids_list_to_tokens(
146
) -> list[str]:
147
"""Detokenize the input ids individually.
148
149
+ Uses decode() for human-readable output, then checks the raw vocab
150
+ piece via convert_ids_to_tokens() to restore any leading spaces that
151
+ decode() stripped (SentencePiece add_dummy_prefix inverse).
152
+
153
Args:
154
tokenizer: tokenizer used by model under test
155
token_ids: convert these tokens (Python list form)
@@ -94,14 +158,16 @@ def convert_ids_list_to_tokens(
158
Python list of token string representations
159
160
"""
97
- token_str_lst = []
98
- for token_id in token_ids:
99
- # use default skip_special_tokens.
100
- token_str = tokenizer.decode([token_id])
101
- if token_str is None:
102
- token_str = ""
103
- token_str_lst.append(token_str)
104
- return token_str_lst
161
+ if not token_ids:
162
+ return []
163
+ marker = _get_leading_space_marker(tokenizer)
164
+ if marker is None:
165
+ return [tokenizer.decode([tid]) or "" for tid in token_ids]
166
+ raw_tokens = tokenizer.convert_ids_to_tokens(token_ids)
167
+ return [
168
+ _restore_leading_spaces(raw, tokenizer.decode([tid]) or "", marker)
169
+ for tid, raw in zip(token_ids, raw_tokens)
170
+ ]
171
172
173
# Based on