A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 03:30 UTC
[Bugfix]: Fix messy code when using logprobs (#19209)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
Chauncey committed
Jul 8, 2025 at 11:02 UTC
93b9d9f499982b723c975ba7066af533afd04f08
3 files changed
+25
-3
tests/test_utils.py
+14
@@ -14,9 +14,12 @@ from unittest.mock import patch
14
import pytest
15
import torch
16
import zmq
17
+from transformers import AutoTokenizer
18
from vllm_test_utils.monitor import monitor
19
20
from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config
21
+from vllm.transformers_utils.detokenizer_utils import (
22
+ convert_ids_list_to_tokens)
23
from vllm.utils import (CacheInfo, FlexibleArgumentParser, LRUCache,
24
MemorySnapshot, PlaceholderModule, StoreBoolean,
25
bind_kv_cache, common_broadcastable_dtype,
@@ -918,3 +921,14 @@ def test_split_host_port():
921
def test_join_host_port():
922
assert join_host_port("127.0.0.1", 5555) == "127.0.0.1:5555"
923
assert join_host_port("::1", 5555) == "[::1]:5555"
924
+
925
+
926
+def test_convert_ids_list_to_tokens():
927
+ tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2.5-1.5B-Instruct")
928
+ token_ids = tokenizer.encode("Hello, world!")
929
+ # token_ids = [9707, 11, 1879, 0]
930
+ assert tokenizer.convert_ids_to_tokens(token_ids) == [
931
+ 'Hello', ',', 'Ġworld', '!'
932
+ ]
933
+ tokens = convert_ids_list_to_tokens(tokenizer, token_ids)
934
+ assert tokens == ['Hello', ',', ' world', '!']
tests/v1/engine/test_output_processor.py
+1
-1
@@ -35,7 +35,7 @@ def _ref_convert_id_to_token(
35
Returns:
36
String representation of input token id
37
"""
38
- return tokenizer.convert_ids_to_tokens(token_id) or ""
38
+ return tokenizer.decode([token_id]) or ""
39
40
41
@pytest.mark.parametrize(
vllm/transformers_utils/detokenizer_utils.py
+10
-2
@@ -78,6 +78,7 @@ def convert_prompt_ids_to_tokens(
78
def convert_ids_list_to_tokens(
79
tokenizer: AnyTokenizer,
80
token_ids: list[int],
81
+ skip_special_tokens: bool = False,
82
) -> list[str]:
83
"""Detokenize the input ids individually.
84
@@ -89,8 +90,15 @@ def convert_ids_list_to_tokens(
90
Python list of token string representations
91
92
"""
92
- token_str_lst = tokenizer.convert_ids_to_tokens(token_ids)
93
- _replace_none_with_empty(token_str_lst) # type: ignore
93
+ token_str_lst = []
94
+ for token_id in token_ids:
95
+ token_str = tokenizer.decode(
96
+ [token_id],
97
+ skip_special_tokens=skip_special_tokens,
98
+ )
99
+ if token_str is None:
100
+ token_str = ""
101
+ token_str_lst.append(token_str)
102
return token_str_lst
103
104