A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 08:30 UTC
[Chore] Move `detokenizer_utils` to `vllm/tokenizers` (#29727)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Nov 29, 2025 at 22:25 UTC
fa59fe417f509641fed102dfa2e3b8a63f224241
9 files changed
+8
-8
tests/tool_use/test_ernie45_moe_tool_parser.py
+1
-1
@@ -15,7 +15,7 @@ from vllm.entrypoints.openai.protocol import (
15
)
16
from vllm.entrypoints.openai.tool_parsers.ernie45_tool_parser import Ernie45ToolParser
17
from vllm.tokenizers import TokenizerLike
18
-from vllm.transformers_utils.detokenizer_utils import detokenize_incrementally
18
+from vllm.tokenizers.detokenizer_utils import detokenize_incrementally
19
from vllm.transformers_utils.tokenizer import get_tokenizer
20
21
# Use a common model that is likely to be available
tests/tool_use/test_jamba_tool_parser.py
+1
-1
@@ -11,7 +11,7 @@ from partial_json_parser.core.options import Allow
11
from vllm.entrypoints.openai.protocol import DeltaMessage, FunctionCall, ToolCall
12
from vllm.entrypoints.openai.tool_parsers.jamba_tool_parser import JambaToolParser
13
from vllm.tokenizers import TokenizerLike
14
-from vllm.transformers_utils.detokenizer_utils import detokenize_incrementally
14
+from vllm.tokenizers.detokenizer_utils import detokenize_incrementally
15
from vllm.transformers_utils.tokenizer import get_tokenizer
16
17
pytestmark = pytest.mark.cpu_test
tests/tool_use/test_qwen3coder_tool_parser.py
+1
-1
@@ -18,7 +18,7 @@ from vllm.entrypoints.openai.tool_parsers.qwen3coder_tool_parser import (
18
)
19
from vllm.entrypoints.openai.tool_parsers.qwen3xml_tool_parser import Qwen3XMLToolParser
20
from vllm.tokenizers import TokenizerLike
21
-from vllm.transformers_utils.detokenizer_utils import detokenize_incrementally
21
+from vllm.tokenizers.detokenizer_utils import detokenize_incrementally
22
from vllm.transformers_utils.tokenizer import get_tokenizer
23
24
pytestmark = pytest.mark.cpu_test
tests/tool_use/test_seed_oss_tool_parser.py
+1
-1
@@ -16,7 +16,7 @@ from vllm.entrypoints.openai.protocol import (
16
)
17
from vllm.entrypoints.openai.tool_parsers.seed_oss_tool_parser import SeedOssToolParser
18
from vllm.tokenizers import TokenizerLike
19
-from vllm.transformers_utils.detokenizer_utils import detokenize_incrementally
19
+from vllm.tokenizers.detokenizer_utils import detokenize_incrementally
20
from vllm.transformers_utils.tokenizer import get_tokenizer
21
22
pytestmark = pytest.mark.cpu_test
tests/tool_use/test_xlam_tool_parser.py
+1
-1
@@ -14,7 +14,7 @@ from vllm.entrypoints.openai.protocol import (
14
)
15
from vllm.entrypoints.openai.tool_parsers.xlam_tool_parser import xLAMToolParser
16
from vllm.tokenizers import TokenizerLike
17
-from vllm.transformers_utils.detokenizer_utils import detokenize_incrementally
17
+from vllm.tokenizers.detokenizer_utils import detokenize_incrementally
18
from vllm.transformers_utils.tokenizer import get_tokenizer
19
20
pytestmark = pytest.mark.cpu_test
tests/utils_/test_argparse_utils.py
+1
-1
@@ -10,7 +10,7 @@ import yaml
10
from transformers import AutoTokenizer
11
from pydantic import ValidationError
12
13
-from vllm.transformers_utils.detokenizer_utils import convert_ids_list_to_tokens
13
+from vllm.tokenizers.detokenizer_utils import convert_ids_list_to_tokens
14
15
from vllm.utils.argparse_utils import FlexibleArgumentParser
16
from ..utils import flat_product
vllm/tokenizers/detokenizer_utils.py
renamed
vllm/v1/engine/detokenizer.py
+1
-1
@@ -9,7 +9,7 @@ from tokenizers.decoders import DecodeStream
9
from transformers import PreTrainedTokenizerFast
10
11
from vllm.logger import init_logger
12
-from vllm.transformers_utils.detokenizer_utils import (
12
+from vllm.tokenizers.detokenizer_utils import (
13
TokenizerLike,
14
convert_prompt_ids_to_tokens,
15
detokenize_incrementally,
vllm/v1/engine/logprobs.py
+1
-1
@@ -12,7 +12,7 @@ from vllm.logprobs import (
12
create_prompt_logprobs,
13
create_sample_logprobs,
14
)
15
-from vllm.transformers_utils.detokenizer_utils import (
15
+from vllm.tokenizers.detokenizer_utils import (
16
TokenizerLike,
17
convert_ids_list_to_tokens,
18
)