A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[RL][BugFix] Fix missing tokenizer error for token-in-token-out (#23904)
Signed-off-by: 22quinn <33176974+22quinn@users.noreply.github.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
22quinn committed
Aug 29, 2025 at 10:09 UTC
4d7fe40fc0468b44404c32d87e4ae0158de24cdc
3 files changed
+99
-18
tests/entrypoints/openai/test_token_in_token_out.py
new
+73
@@ -0,0 +1,73 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+import os
5
+import tempfile
6
+
7
+import pytest
8
+
9
+from vllm.model_executor.model_loader.weight_utils import (
10
+ download_weights_from_hf)
11
+from vllm.transformers_utils.tokenizer import get_tokenizer
12
+
13
+from ...utils import RemoteOpenAIServer
14
+
15
+MODEL_NAME = "Qwen/Qwen3-0.6B"
16
+MODEL_PATH = os.path.join(tempfile.gettempdir(), "qwen3_06b")
17
+
18
+
19
+@pytest.fixture(scope="module")
20
+def server():
21
+ global MODEL_PATH
22
+ MODEL_PATH = download_weights_from_hf(
23
+ MODEL_NAME,
24
+ allow_patterns=["*"],
25
+ cache_dir=MODEL_PATH,
26
+ ignore_patterns=["tokenizer*", "vocab*", "*.safetensors"])
27
+ args = [
28
+ "--max-model-len",
29
+ "2048",
30
+ "--max-num-seqs",
31
+ "128",
32
+ "--enforce-eager",
33
+ "--skip-tokenizer-init",
34
+ "--load-format",
35
+ "dummy",
36
+ ]
37
+ with RemoteOpenAIServer(MODEL_PATH, args) as remote_server:
38
+ yield remote_server
39
+
40
+
41
+@pytest.mark.asyncio
42
+async def test_token_in_token_out_and_logprobs(server):
43
+ """
44
+ Test token-in-token-out and token_ids align with prompt_logprobs
45
+ & logprobs when return_tokens_as_token_ids is enabled.
46
+ """
47
+ tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
48
+ text = "Hello, world! How are you today?"
49
+ token_ids = tokenizer.encode(text)
50
+ async with server.get_async_client() as client:
51
+ # Test with both return_token_ids and return_tokens_as_token_ids enabled
52
+ completion = await client.completions.create(
53
+ model=MODEL_PATH,
54
+ prompt=token_ids,
55
+ max_tokens=20,
56
+ temperature=0,
57
+ echo=True,
58
+ extra_body={
59
+ "return_token_ids": True,
60
+ },
61
+ )
62
+
63
+ # Verify all fields are present
64
+ assert (completion.choices[0].token_ids is not None
65
+ and 0 < len(completion.choices[0].token_ids) <= 20)
66
+ assert completion.choices[0].prompt_token_ids is not None
67
+
68
+ # Decode prompt tokens
69
+ if completion.choices[0].prompt_token_ids:
70
+ prompt_text = tokenizer.decode(
71
+ completion.choices[0].prompt_token_ids)
72
+ # The decoded prompt should match or close to original prompt
73
+ assert prompt_text == text
vllm/entrypoints/openai/serving_completion.py
+5
-1
@@ -127,7 +127,11 @@ class OpenAIServingCompletion(OpenAIServing):
127
try:
128
lora_request = self._maybe_get_adapters(request)
129
130
- tokenizer = await self.engine_client.get_tokenizer(lora_request)
130
+ if self.model_config.skip_tokenizer_init:
131
+ tokenizer = None
132
+ else:
133
+ tokenizer = await self.engine_client.get_tokenizer(lora_request
134
+ )
135
136
request_prompts, engine_prompts = await self._preprocess_completion(
137
request,
vllm/entrypoints/openai/serving_engine.py
+21
-17
@@ -526,8 +526,8 @@ class OpenAIServing:
526
async def _normalize_prompt_text_to_input(
527
self,
528
request: AnyRequest,
529
- tokenizer: AnyTokenizer,
529
prompt: str,
530
+ tokenizer: AnyTokenizer,
531
truncate_prompt_tokens: Optional[Annotated[int, Field(ge=-1)]],
532
add_special_tokens: bool,
533
) -> TextTokensPrompt:
@@ -563,12 +563,10 @@ class OpenAIServing:
563
async def _normalize_prompt_tokens_to_input(
564
self,
565
request: AnyRequest,
566
- tokenizer: AnyTokenizer,
566
prompt_ids: list[int],
567
+ tokenizer: Optional[AnyTokenizer],
568
truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]],
569
) -> TextTokensPrompt:
570
- async_tokenizer = self._get_async_tokenizer(tokenizer)
571
-
570
if truncate_prompt_tokens is None:
571
input_ids = prompt_ids
572
elif truncate_prompt_tokens < 0:
@@ -576,7 +574,11 @@ class OpenAIServing:
574
else:
575
input_ids = prompt_ids[-truncate_prompt_tokens:]
576
579
- input_text = await async_tokenizer.decode(input_ids)
577
+ if tokenizer is None:
578
+ input_text = ""
579
+ else:
580
+ async_tokenizer = self._get_async_tokenizer(tokenizer)
581
+ input_text = await async_tokenizer.decode(input_ids)
582
583
return self._validate_input(request, input_ids, input_text)
584
@@ -681,27 +683,27 @@ class OpenAIServing:
683
[`_tokenize_prompt_input_or_inputs`][vllm.entrypoints.openai.serving_engine.OpenAIServing._tokenize_prompt_input_or_inputs]
684
that assumes multiple inputs.
685
"""
684
- for text in prompt_inputs:
685
- if isinstance(text, str):
686
+ for prompt in prompt_inputs:
687
+ if isinstance(prompt, str):
688
yield await self._normalize_prompt_text_to_input(
689
request,
688
- tokenizer,
689
- prompt=text,
690
+ prompt=prompt,
691
+ tokenizer=tokenizer,
692
truncate_prompt_tokens=truncate_prompt_tokens,
693
add_special_tokens=add_special_tokens,
694
)
695
else:
696
yield await self._normalize_prompt_tokens_to_input(
697
request,
696
- tokenizer,
697
- prompt_ids=text,
698
+ prompt_ids=prompt,
699
+ tokenizer=tokenizer,
700
truncate_prompt_tokens=truncate_prompt_tokens,
701
)
702
703
async def _tokenize_prompt_input_or_inputs_async(
704
self,
705
request: AnyRequest,
704
- tokenizer: AnyTokenizer,
706
+ tokenizer: Optional[AnyTokenizer],
707
input_or_inputs: Optional[Union[str, list[str], list[int],
708
list[list[int]]]],
709
truncate_prompt_tokens: Optional[Annotated[int, Field(ge=-1)]] = None,
@@ -740,17 +742,19 @@ class OpenAIServing:
742
tasks = []
743
for prompt_input in batch_inputs:
744
if prompt_input["is_tokens"] is False:
745
+ assert tokenizer is not None, \
746
+ "Tokenizer is required for text prompts"
747
task = self._normalize_prompt_text_to_input(
748
request,
745
- tokenizer,
749
prompt_input["content"],
750
+ tokenizer=tokenizer,
751
truncate_prompt_tokens=truncate_prompt_tokens,
752
add_special_tokens=add_special_tokens)
753
else:
754
task = self._normalize_prompt_tokens_to_input(
755
request,
752
- tokenizer,
756
prompt_input["content"],
757
+ tokenizer=tokenizer,
758
truncate_prompt_tokens=truncate_prompt_tokens)
759
tasks.append(task)
760
@@ -766,7 +770,7 @@ class OpenAIServing:
770
request: Union[DetokenizeRequest, EmbeddingCompletionRequest,
771
RerankRequest, ClassificationRequest, ScoreRequest,
772
TokenizeCompletionRequest],
769
- tokenizer: AnyTokenizer,
773
+ tokenizer: Optional[AnyTokenizer],
774
input_or_inputs: Union[str, list[str], list[int], list[list[int]]],
775
truncate_prompt_tokens: Optional[Annotated[int, Field(ge=-1)]] = ...,
776
add_special_tokens: bool = ...,
@@ -777,7 +781,7 @@ class OpenAIServing:
781
async def _preprocess_completion(
782
self,
783
request: CompletionRequest,
780
- tokenizer: AnyTokenizer,
784
+ tokenizer: Optional[AnyTokenizer],
785
input_or_inputs: Optional[Union[str, list[str], list[int],
786
list[list[int]]]],
787
truncate_prompt_tokens: Optional[Annotated[int, Field(ge=-1)]] = ...,
@@ -789,7 +793,7 @@ class OpenAIServing:
793
async def _preprocess_completion(
794
self,
795
request: CompletionLikeRequest,
792
- tokenizer: AnyTokenizer,
796
+ tokenizer: Optional[AnyTokenizer],
797
input_or_inputs: Optional[Union[str, list[str], list[int],
798
list[list[int]]]],
799
truncate_prompt_tokens: Optional[Annotated[int, Field(ge=-1)]] = None,