@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d117a4d1a9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 9, 19:30 UTC

[Frontend] Introduce Renderer for processing chat messages (using `ModelConfig`) (#30200)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Jan 22, 2026 at 20:44 UTC d117a4d1a915c7acf124563f9b337de5c0aa2c2f
48 files changed +2141 -1585
.buildkite/test-amd.yaml
+2
@@ -71,6 +71,7 @@ steps:
71 - tests/test_inputs.py
72 - tests/test_outputs.py
73 - tests/multimodal
74 + - tests/renderers
75 - tests/standalone_tests/lazy_imports.py
76 - tests/tokenizers_
77 - tests/tool_parsers
@@ -82,6 +83,7 @@ steps:
83 - pytest -v -s test_inputs.py
84 - pytest -v -s test_outputs.py
85 - pytest -v -s -m 'cpu_test' multimodal
86 + - pytest -v -s renderers
87 - pytest -v -s tokenizers_
88 - pytest -v -s tool_parsers
89 - pytest -v -s transformers_utils
.buildkite/test-pipeline.yaml
+2
@@ -64,6 +64,7 @@ steps:
64 - tests/test_inputs.py
65 - tests/test_outputs.py
66 - tests/multimodal
67 + - tests/renderers
68 - tests/standalone_tests/lazy_imports.py
69 - tests/tokenizers_
70 - tests/tool_parsers
@@ -75,6 +76,7 @@ steps:
76 - pytest -v -s test_inputs.py
77 - pytest -v -s test_outputs.py
78 - pytest -v -s -m 'cpu_test' multimodal
79 + - pytest -v -s renderers
80 - pytest -v -s tokenizers_
81 - pytest -v -s tool_parsers
82 - pytest -v -s transformers_utils
.buildkite/test_areas/misc.yaml
+2
@@ -121,6 +121,7 @@ steps:
121 - tests/test_inputs.py
122 - tests/test_outputs.py
123 - tests/multimodal
124 + - tests/renderers
125 - tests/standalone_tests/lazy_imports.py
126 - tests/tokenizers_
127 - tests/tool_parsers
@@ -132,6 +133,7 @@ steps:
133 - pytest -v -s test_inputs.py
134 - pytest -v -s test_outputs.py
135 - pytest -v -s -m 'cpu_test' multimodal
136 + - pytest -v -s renderers
137 - pytest -v -s tokenizers_
138 - pytest -v -s tool_parsers
139 - pytest -v -s transformers_utils
docs/features/reasoning_outputs.md
+2 -1
@@ -254,7 +254,8 @@ You can add a new `ReasoningParser` similar to [vllm/reasoning/deepseek_r1_reaso
254 # import the required packages
255
256 from vllm.reasoning import ReasoningParser, ReasoningParserManager
257 - from vllm.entrypoints.openai.protocol import ChatCompletionRequest, DeltaMessage
257 + from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
258 + from vllm.entrypoints.openai.engine.protocol import DeltaMessage
259
260 # define a reasoning parser and register it to vllm
261 # the name list in register_module can be used
tests/entrypoints/openai/test_chat_template.py deleted
-156
@@ -1,156 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -
4 -import pytest
5 -
6 -from vllm.config import ModelConfig
7 -from vllm.entrypoints.chat_utils import apply_hf_chat_template, load_chat_template
8 -from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
9 -from vllm.tokenizers import get_tokenizer
10 -
11 -from ...models.registry import HF_EXAMPLE_MODELS
12 -from ...utils import VLLM_PATH
13 -
14 -chatml_jinja_path = VLLM_PATH / "examples/template_chatml.jinja"
15 -assert chatml_jinja_path.exists()
16 -
17 -# Define models, templates, and their corresponding expected outputs
18 -MODEL_TEMPLATE_GENERATION_OUTPUT = [
19 - (
20 - "facebook/opt-125m",
21 - chatml_jinja_path,
22 - True,
23 - False,
24 - """<|im_start|>user
25 -Hello<|im_end|>
26 -<|im_start|>assistant
27 -Hi there!<|im_end|>
28 -<|im_start|>user
29 -What is the capital of<|im_end|>
30 -<|im_start|>assistant
31 -""",
32 - ),
33 - (
34 - "facebook/opt-125m",
35 - chatml_jinja_path,
36 - False,
37 - False,
38 - """<|im_start|>user
39 -Hello<|im_end|>
40 -<|im_start|>assistant
41 -Hi there!<|im_end|>
42 -<|im_start|>user
43 -What is the capital of""",
44 - ),
45 - (
46 - "facebook/opt-125m",
47 - chatml_jinja_path,
48 - False,
49 - True,
50 - """<|im_start|>user
51 -Hello<|im_end|>
52 -<|im_start|>assistant
53 -Hi there!<|im_end|>
54 -<|im_start|>user
55 -What is the capital of<|im_end|>
56 -<|im_start|>assistant
57 -The capital of""",
58 - ),
59 -]
60 -
61 -TEST_MESSAGES = [
62 - {"role": "user", "content": "Hello"},
63 - {"role": "assistant", "content": "Hi there!"},
64 - {"role": "user", "content": "What is the capital of"},
65 -]
66 -ASSISTANT_MESSAGE_TO_CONTINUE = {"role": "assistant", "content": "The capital of"}
67 -
68 -
69 -def test_load_chat_template():
70 - # Testing chatml template
71 - template_content = load_chat_template(chat_template=chatml_jinja_path)
72 -
73 - # Test assertions
74 - assert template_content is not None
75 - # Hard coded value for template_chatml.jinja
76 - assert (
77 - template_content
78 - == """{% for message in messages %}{{'<|im_start|>' + message['role'] + '\\n' + message['content']}}{% if (loop.last and add_generation_prompt) or not loop.last %}{{ '<|im_end|>' + '\\n'}}{% endif %}{% endfor %}
79 -{% if add_generation_prompt and messages[-1]['role'] != 'assistant' %}{{ '<|im_start|>assistant\\n' }}{% endif %}""" # noqa: E501
80 - )
81 -
82 -
83 -def test_no_load_chat_template_filelike():
84 - # Testing chatml template
85 - template = "../../examples/does_not_exist"
86 -
87 - with pytest.raises(ValueError, match="looks like a file path"):
88 - load_chat_template(chat_template=template)
89 -
90 -
91 -def test_no_load_chat_template_literallike():
92 - # Testing chatml template
93 - template = "{{ messages }}"
94 -
95 - template_content = load_chat_template(chat_template=template)
96 -
97 - assert template_content == template
98 -
99 -
100 -@pytest.mark.parametrize(
101 - "model,template,add_generation_prompt,continue_final_message,expected_output",
102 - MODEL_TEMPLATE_GENERATION_OUTPUT,
103 -)
104 -def test_get_gen_prompt(
105 - model, template, add_generation_prompt, continue_final_message, expected_output
106 -):
107 - model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
108 - model_info.check_available_online(on_fail="skip")
109 -
110 - model_config = ModelConfig(
111 - model,
112 - tokenizer=model_info.tokenizer or model,
113 - tokenizer_mode=model_info.tokenizer_mode,
114 - trust_remote_code=model_info.trust_remote_code,
115 - revision=model_info.revision,
116 - hf_overrides=model_info.hf_overrides,
117 - skip_tokenizer_init=model_info.require_embed_inputs,
118 - enable_prompt_embeds=model_info.require_embed_inputs,
119 - enable_mm_embeds=model_info.require_embed_inputs,
120 - enforce_eager=model_info.enforce_eager,
121 - dtype=model_info.dtype,
122 - )
123 -
124 - # Initialize the tokenizer
125 - tokenizer = get_tokenizer(
126 - tokenizer_name=model_config.tokenizer,
127 - trust_remote_code=model_config.trust_remote_code,
128 - )
129 - template_content = load_chat_template(chat_template=template)
130 -
131 - # Create a mock request object using keyword arguments
132 - mock_request = ChatCompletionRequest(
133 - model=model,
134 - messages=TEST_MESSAGES + [ASSISTANT_MESSAGE_TO_CONTINUE]
135 - if continue_final_message
136 - else TEST_MESSAGES,
137 - add_generation_prompt=add_generation_prompt,
138 - continue_final_message=continue_final_message,
139 - )
140 -
141 - # Call the function and get the result
142 - result = apply_hf_chat_template(
143 - tokenizer=tokenizer,
144 - conversation=mock_request.messages,
145 - chat_template=mock_request.chat_template or template_content,
146 - model_config=model_config,
147 - tools=None,
148 - add_generation_prompt=mock_request.add_generation_prompt,
149 - continue_final_message=mock_request.continue_final_message,
150 - )
151 -
152 - # Test assertion
153 - assert result == expected_output, (
154 - f"The generated prompt does not match the expected output for "
155 - f"model {model} and template {template}"
156 - )
tests/entrypoints/openai/test_serving_chat.py
+58 -58
@@ -11,7 +11,7 @@ import pytest_asyncio
11 from openai import OpenAI
12
13 from vllm._aiter_ops import is_aiter_found_and_supported
14 -from vllm.config.multimodal import MultiModalConfig
14 +from vllm.config import MultiModalConfig
15 from vllm.entrypoints.openai.chat_completion.protocol import (
16 ChatCompletionRequest,
17 ChatCompletionResponse,
@@ -23,8 +23,13 @@ from vllm.entrypoints.openai.engine.protocol import (
23 )
24 from vllm.entrypoints.openai.models.serving import BaseModelPath, OpenAIServingModels
25 from vllm.entrypoints.openai.parser.harmony_utils import get_encoding
26 +from vllm.inputs import TokensPrompt
27 from vllm.outputs import CompletionOutput, RequestOutput
28 +from vllm.renderers.hf import HfRenderer
29 +from vllm.renderers.mistral import MistralRenderer
30 from vllm.tokenizers import get_tokenizer
31 +from vllm.tokenizers.mistral import MistralTokenizer
32 +from vllm.tokenizers.registry import tokenizer_args_from_config
33 from vllm.tool_parsers import ToolParserManager
34 from vllm.v1.engine.async_llm import AsyncLLM
35
@@ -103,15 +108,16 @@ def gptoss_server(default_server_args: list[str]):
108
109 @pytest.fixture(scope="class")
110 def gptoss_speculative_server(default_server_args: list[str]):
111 + attention_backend = (
112 + "TRITON_ATTN"
113 + if not is_aiter_found_and_supported()
114 + else "ROCM_AITER_UNIFIED_ATTN"
115 + )
116 server_args = default_server_args + [
117 "--speculative-config",
118 f'{{"model": "{GPT_OSS_SPECULATOR_NAME}", '
119 f'"method": "eagle3", "num_speculative_tokens": 3}}',
110 - f"--attention-backend={
111 - 'TRITON_ATTN'
112 - if not is_aiter_found_and_supported()
113 - else 'ROCM_AITER_UNIFIED_ATTN'
114 - }",
120 + f"--attention-backend={attention_backend}",
121 ]
122 # gpt-oss requires AITER unified attention on ROCm
123 # TODO: Remove after fixing TRITON_ATTN issue on ROCm
@@ -520,12 +526,21 @@ class MockModelConfig:
526 encoder_config = None
527 generation_config: str = "auto"
528 media_io_kwargs: dict[str, dict[str, Any]] = field(default_factory=dict)
523 - skip_tokenizer_init = False
529 + skip_tokenizer_init: bool = False
530
531 def get_diff_sampling_param(self):
532 return self.diff_sampling_param or {}
533
534
535 +def _build_renderer(model_config: MockModelConfig):
536 + _, tokenizer_name, _, kwargs = tokenizer_args_from_config(model_config)
537 +
538 + return HfRenderer(
539 + model_config,
540 + tokenizer_kwargs={**kwargs, "tokenizer_name": tokenizer_name},
541 + )
542 +
543 +
544 def _build_serving_chat(engine: AsyncLLM) -> OpenAIServingChat:
545 models = OpenAIServingModels(
546 engine_client=engine,
@@ -561,6 +576,7 @@ class MockEngine:
576 model_config: MockModelConfig = field(default_factory=MockModelConfig)
577 input_processor: MagicMock = field(default_factory=MagicMock)
578 io_processor: MagicMock = field(default_factory=MagicMock)
579 + renderer: MagicMock = field(default_factory=MagicMock)
580
581
582 async def _async_serving_chat_init():
@@ -586,11 +602,11 @@ def test_async_serving_chat_init():
602 @pytest.mark.asyncio
603 async def test_serving_chat_returns_correct_model_name():
604 mock_engine = MagicMock(spec=AsyncLLM)
589 - mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME)
605 mock_engine.errored = False
606 mock_engine.model_config = MockModelConfig()
607 mock_engine.input_processor = MagicMock()
608 mock_engine.io_processor = MagicMock()
609 + mock_engine.renderer = _build_renderer(mock_engine.model_config)
610
611 serving_chat = _build_serving_chat(mock_engine)
612 messages = [{"role": "user", "content": "what is 1+1?"}]
@@ -616,11 +632,11 @@ async def test_serving_chat_returns_correct_model_name():
632 @pytest.mark.asyncio
633 async def test_serving_chat_should_set_correct_max_tokens():
634 mock_engine = MagicMock(spec=AsyncLLM)
619 - mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME)
635 mock_engine.errored = False
636 mock_engine.model_config = MockModelConfig()
637 mock_engine.input_processor = MagicMock()
638 mock_engine.io_processor = MagicMock()
639 + mock_engine.renderer = _build_renderer(mock_engine.model_config)
640
641 serving_chat = _build_serving_chat(mock_engine)
642
@@ -649,11 +665,11 @@ async def test_serving_chat_should_set_correct_max_tokens():
665
666 # Reinitialize the engine with new settings
667 mock_engine = MagicMock(spec=AsyncLLM)
652 - mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME)
668 mock_engine.errored = False
669 mock_engine.model_config = mock_model_config
670 mock_engine.input_processor = MagicMock()
671 mock_engine.io_processor = MagicMock()
672 + mock_engine.renderer = _build_renderer(mock_engine.model_config)
673
674 # Initialize the serving chat
675 serving_chat = _build_serving_chat(mock_engine)
@@ -694,11 +710,11 @@ async def test_serving_chat_should_set_correct_max_tokens():
710
711 # Reinitialize the engine with new settings
712 mock_engine = MagicMock(spec=AsyncLLM)
697 - mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME)
713 mock_engine.errored = False
714 mock_engine.model_config = mock_model_config
715 mock_engine.input_processor = MagicMock()
716 mock_engine.io_processor = MagicMock()
717 + mock_engine.renderer = _build_renderer(mock_engine.model_config)
718
719 # Initialize the serving chat
720 serving_chat = _build_serving_chat(mock_engine)
@@ -732,42 +748,32 @@ async def test_serving_chat_should_set_correct_max_tokens():
748
749
750 @pytest.mark.asyncio
735 -async def test_serving_chat_mistral_token_ids_prompt_is_validated(monkeypatch_module):
751 +async def test_serving_chat_mistral_token_ids_prompt_is_validated():
752 """Regression test: when the Mistral tokenizer path returns token IDs
753 directly, we must still apply input length + max_tokens validation.
754 """
755
756 mock_engine = MagicMock(spec=AsyncLLM)
757 mock_engine.errored = False
742 - mock_engine.model_config = MockModelConfig()
758 + mock_engine.model_config = MockModelConfig(skip_tokenizer_init=True)
759 mock_engine.input_processor = MagicMock()
760 mock_engine.io_processor = MagicMock()
761
746 - class DummyMistralTokenizer:
747 - def decode(self, token_ids):
748 - # Only used for logging/validation error messages.
749 - return "dummy"
750 -
751 - dummy_tokenizer = DummyMistralTokenizer()
752 - mock_engine.get_tokenizer.return_value = dummy_tokenizer
753 -
754 - # Patch the OpenAI engine serving module to treat our dummy tokenizer
755 - # as a MistralTokenizer. This forces the code path where chat template
756 - # rendering can return a list[int] (token IDs).
757 - import vllm.entrypoints.openai.engine.serving as engine_serving
758 -
759 - monkeypatch_module.setattr(
760 - engine_serving, "MistralTokenizer", DummyMistralTokenizer
761 - )
762 -
763 - serving_chat = _build_serving_chat(mock_engine)
764 -
762 + mock_tokenizer = MagicMock(spec=MistralTokenizer)
763 + mock_renderer = MistralRenderer(mock_engine.model_config, tokenizer_kwargs={})
764 + mock_renderer._tokenizer = mock_tokenizer
765 # Force the Mistral chat template renderer to return token IDs.
766 # Choose a prompt length that is < max_model_len, but large enough that
767 # adding max_tokens should exceed the model context window.
768 - serving_chat._apply_mistral_chat_template_async = AsyncMock(
769 - return_value=list(range(95))
768 + mock_renderer.render_messages_async = AsyncMock(
769 + return_value=(
770 + [],
771 + TokensPrompt(prompt_token_ids=list(range(95))),
772 + )
773 )
774 + mock_engine.renderer = mock_renderer
775 +
776 + serving_chat = _build_serving_chat(mock_engine)
777
778 req = ChatCompletionRequest(
779 model=MODEL_NAME,
@@ -781,39 +787,33 @@ async def test_serving_chat_mistral_token_ids_prompt_is_validated(monkeypatch_mo
787
788
789 @pytest.mark.asyncio
784 -async def test_serving_chat_mistral_token_ids_prompt_too_long_is_rejected(
785 - monkeypatch_module,
786 -):
790 +async def test_serving_chat_mistral_token_ids_prompt_too_long_is_rejected():
791 """Regression test: MistralTokenizer token-id prompts must still enforce
792 the max context length for the input itself (token_num >= max_model_len).
793 """
794
795 mock_engine = MagicMock(spec=AsyncLLM)
796 mock_engine.errored = False
793 - mock_engine.model_config = MockModelConfig()
797 + mock_engine.model_config = MockModelConfig(skip_tokenizer_init=True)
798 mock_engine.input_processor = MagicMock()
799 mock_engine.io_processor = MagicMock()
800
797 - class DummyMistralTokenizer:
798 - def decode(self, token_ids):
799 - return "dummy"
800 -
801 - dummy_tokenizer = DummyMistralTokenizer()
802 - mock_engine.get_tokenizer.return_value = dummy_tokenizer
803 -
804 - import vllm.entrypoints.openai.engine.serving as engine_serving
805 -
806 - monkeypatch_module.setattr(
807 - engine_serving, "MistralTokenizer", DummyMistralTokenizer
808 - )
809 -
810 - serving_chat = _build_serving_chat(mock_engine)
811 -
801 + mock_tokenizer = MagicMock(spec=MistralTokenizer)
802 + mock_renderer = MistralRenderer(mock_engine.model_config, tokenizer_kwargs={})
803 + mock_renderer._tokenizer = mock_tokenizer
804 # prompt_token_ids length == max_model_len should be rejected for
805 # completion-like requests (ChatCompletionRequest).
814 - serving_chat._apply_mistral_chat_template_async = AsyncMock(
815 - return_value=list(range(mock_engine.model_config.max_model_len))
806 + mock_renderer.render_messages_async = AsyncMock(
807 + return_value=(
808 + [],
809 + TokensPrompt(
810 + prompt_token_ids=list(range(mock_engine.model_config.max_model_len))
811 + ),
812 + )
813 )
814 + mock_engine.renderer = mock_renderer
815 +
816 + serving_chat = _build_serving_chat(mock_engine)
817
818 req = ChatCompletionRequest(
819 model=MODEL_NAME,
@@ -835,11 +835,11 @@ async def test_serving_chat_could_load_correct_generation_config():
835 }
836
837 mock_engine = MagicMock(spec=AsyncLLM)
838 - mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME)
838 mock_engine.errored = False
839 mock_engine.model_config = mock_model_config
840 mock_engine.input_processor = MagicMock()
841 mock_engine.io_processor = MagicMock()
842 + mock_engine.renderer = _build_renderer(mock_engine.model_config)
843
844 # Initialize the serving chat
845 serving_chat = _build_serving_chat(mock_engine)
@@ -881,11 +881,11 @@ async def test_serving_chat_did_set_correct_cache_salt(model_type):
881 mock_model_config.hf_config.model_type = model_type
882
883 mock_engine = MagicMock(spec=AsyncLLM)
884 - mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME)
884 mock_engine.errored = False
885 mock_engine.model_config = mock_model_config
886 mock_engine.input_processor = MagicMock()
887 mock_engine.io_processor = MagicMock()
888 + mock_engine.renderer = _build_renderer(mock_engine.model_config)
889
890 serving_chat = _build_serving_chat(mock_engine)
891
@@ -914,11 +914,11 @@ async def test_serving_chat_data_parallel_rank_extraction():
914 """Test that data_parallel_rank is properly extracted from header and
915 passed to engine."""
916 mock_engine = MagicMock(spec=AsyncLLM)
917 - mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME)
917 mock_engine.errored = False
918 mock_engine.model_config = MockModelConfig()
919 mock_engine.input_processor = MagicMock()
920 mock_engine.io_processor = MagicMock()
921 + mock_engine.renderer = _build_renderer(mock_engine.model_config)
922
923 # Mock the generate method to return an async generator
924 async def mock_generate(*args, **kwargs):
tests/entrypoints/openai/test_serving_engine.py deleted
-71
@@ -1,71 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -
4 -import asyncio
5 -import time
6 -from unittest.mock import Mock
7 -
8 -import pytest
9 -
10 -from vllm.config import ModelConfig
11 -from vllm.entrypoints.openai.engine.serving import OpenAIServing
12 -from vllm.entrypoints.openai.models.serving import OpenAIServingModels
13 -from vllm.tokenizers.mistral import MistralTokenizer
14 -
15 -
16 -@pytest.fixture()
17 -def serving() -> OpenAIServing:
18 - """Create a minimal OpenAIServing instance for testing."""
19 -
20 - # Create minimal mocks
21 - engine_client = Mock()
22 - model_config = Mock(spec=ModelConfig)
23 - model_config.max_model_len = 32768
24 - models = Mock(spec=OpenAIServingModels)
25 - models.model_config = model_config
26 - models.input_processor = Mock()
27 - models.io_processor = Mock()
28 -
29 - serving = OpenAIServing(
30 - engine_client=engine_client,
31 - models=models,
32 - request_logger=None,
33 - )
34 - return serving
35 -
36 -
37 -@pytest.mark.asyncio
38 -async def test_async_mistral_tokenizer_does_not_block_event_loop(
39 - serving: OpenAIServing,
40 -):
41 - expected_tokens = [1, 2, 3]
42 -
43 - # Mock the blocking version to sleep
44 - def mocked_apply_chat_template(*_args, **_kwargs):
45 - time.sleep(2)
46 - return expected_tokens
47 -
48 - mock_tokenizer = Mock(spec=MistralTokenizer)
49 - mock_tokenizer.apply_chat_template.side_effect = mocked_apply_chat_template
50 -
51 - task = serving._apply_mistral_chat_template_async(
52 - tokenizer=mock_tokenizer, messages=[], chat_template=None, tools=[]
53 - )
54 -
55 - # Ensure the event loop is not blocked
56 - blocked_count = 0
57 - for _i in range(20): # Check over ~2 seconds
58 - start = time.perf_counter()
59 - await asyncio.sleep(0)
60 - elapsed = time.perf_counter() - start
61 -
62 - # an overly generous elapsed time for slow machines
63 - if elapsed >= 0.5:
64 - blocked_count += 1
65 -
66 - await asyncio.sleep(0.1)
67 -
68 - # Ensure task completes
69 - tokens = await task
70 - assert tokens == expected_tokens, "Mocked blocking tokenizer was not called"
71 - assert blocked_count == 0, "Event loop blocked during tokenization"
tests/entrypoints/openai/test_serving_models.py
+1
@@ -35,6 +35,7 @@ async def _async_serving_models_init() -> OpenAIServingModels:
35 mock_engine_client.model_config = mock_model_config
36 mock_engine_client.input_processor = MagicMock()
37 mock_engine_client.io_processor = MagicMock()
38 + mock_engine_client.renderer = MagicMock()
39
40 serving_models = OpenAIServingModels(
41 engine_client=mock_engine_client,
tests/entrypoints/openai/test_serving_responses.py
+2
@@ -131,6 +131,7 @@ class TestInitializeToolSessions:
131
132 engine_client.input_processor = MagicMock()
133 engine_client.io_processor = MagicMock()
134 + engine_client.renderer = MagicMock()
135
136 models = MagicMock()
137
@@ -217,6 +218,7 @@ class TestValidateGeneratorInput:
218
219 engine_client.input_processor = MagicMock()
220 engine_client.io_processor = MagicMock()
221 + engine_client.renderer = MagicMock()
222
223 models = MagicMock()
224
tests/entrypoints/pooling/score/test_utils.py
+4 -4
@@ -212,7 +212,7 @@ class TestGetScorePrompt:
212 return_value=mock_model_no_score_template,
213 ),
214 patch(
215 - "vllm.entrypoints.pooling.score.utils.apply_hf_chat_template",
215 + "vllm.entrypoints.pooling.score.utils.safe_apply_chat_template",
216 return_value="test querytest doc",
217 ),
218 ):
@@ -245,7 +245,7 @@ class TestGetScorePrompt:
245 return_value=mock_model_no_score_template,
246 ),
247 patch(
248 - "vllm.entrypoints.pooling.score.utils.apply_hf_chat_template",
248 + "vllm.entrypoints.pooling.score.utils.safe_apply_chat_template",
249 side_effect=ChatTemplateResolutionError("No template"),
250 ),
251 ):
@@ -296,7 +296,7 @@ class TestGetScorePrompt:
296 return_value=mock_model_no_score_template,
297 ),
298 patch(
299 - "vllm.entrypoints.pooling.score.utils.apply_hf_chat_template",
299 + "vllm.entrypoints.pooling.score.utils.safe_apply_chat_template",
300 side_effect=ChatTemplateResolutionError("No template"),
301 ),
302 ):
@@ -331,7 +331,7 @@ class TestGetScorePrompt:
331 return_value=mock_model_with_score_template,
332 ),
333 patch(
334 - "vllm.entrypoints.pooling.score.utils.apply_hf_chat_template",
334 + "vllm.entrypoints.pooling.score.utils.safe_apply_chat_template",
335 side_effect=ChatTemplateResolutionError("No template"),
336 ),
337 ):
tests/entrypoints/test_chat_utils.py
+24 -468
@@ -7,21 +7,14 @@ from typing import Literal
7
8 import pytest
9 import torch
10 -from mistral_common.tokens.tokenizers.base import SpecialTokenPolicy
10
11 from vllm.assets.audio import AudioAsset
12 from vllm.assets.image import ImageAsset
13 from vllm.assets.video import VideoAsset
14 from vllm.config import ModelConfig
15 from vllm.entrypoints.chat_utils import (
17 - _try_extract_ast,
18 - apply_mistral_chat_template,
19 - load_chat_template,
16 parse_chat_messages,
21 - parse_chat_messages_futures,
22 - resolve_chat_template_content_format,
23 - resolve_chat_template_kwargs,
24 - resolve_hf_chat_template,
17 + parse_chat_messages_async,
18 )
19 from vllm.multimodal import MultiModalDataDict, MultiModalUUIDDict
20 from vllm.multimodal.utils import (
@@ -29,24 +22,11 @@ from vllm.multimodal.utils import (
22 encode_image_url,
23 encode_video_url,
24 )
32 -from vllm.tokenizers import get_tokenizer
33 -from vllm.tokenizers.mistral import MistralTokenizer
25 from vllm.utils.serial_utils import tensor2base64
26
36 -from ..models.registry import HF_EXAMPLE_MODELS
37 -from ..utils import VLLM_PATH
38 -
39 -EXAMPLES_DIR = VLLM_PATH / "examples"
40 -
27 PHI3V_MODEL_ID = "microsoft/Phi-3.5-vision-instruct"
42 -ULTRAVOX_MODEL_ID = "fixie-ai/ultravox-v0_5-llama-3_2-1b"
28 QWEN2AUDIO_MODEL_ID = "Qwen/Qwen2-Audio-7B-Instruct"
44 -QWEN2VL_MODEL_ID = "Qwen/Qwen2-VL-2B-Instruct"
45 -QWEN25VL_MODEL_ID = "Qwen/Qwen2.5-VL-3B-Instruct"
29 QWEN25OMNI_MODEL_ID = "Qwen/Qwen2.5-Omni-7B"
47 -QWEN3_MODEL_ID = "Qwen/Qwen3-8B"
48 -LLAMA_GUARD_MODEL_ID = "meta-llama/Llama-Guard-3-1B"
49 -HERMES_MODEL_ID = "NousResearch/Hermes-3-Llama-3.1-8B"
30 MISTRAL_MODEL_ID = "mistralai/Mistral-Small-3.1-24B-Instruct-2503"
31
32
@@ -469,7 +449,7 @@ async def test_parse_chat_messages_single_image_with_uuid_async(
449 image_url,
450 ):
451 image_uuid = str(hash(image_url))
472 - conversation, mm_future, mm_uuids = parse_chat_messages_futures(
452 + conversation, mm_data, mm_uuids = await parse_chat_messages_async(
453 [
454 {
455 "role": "user",
@@ -490,7 +470,7 @@ async def test_parse_chat_messages_single_image_with_uuid_async(
470 assert conversation == [
471 {"role": "user", "content": "<|image_1|>\nWhat's in the image?"}
472 ]
493 - _assert_mm_data_is_image_input(await mm_future, 1)
473 + _assert_mm_data_is_image_input(mm_data, 1)
474 _assert_mm_uuids(mm_uuids, 1, expected_uuids=[image_uuid])
475
476
@@ -500,7 +480,7 @@ async def test_parse_chat_messages_empty_image_with_uuid_async(
480 image_url,
481 ):
482 image_uuid = str(hash(image_url))
503 - conversation, mm_future, mm_uuids = parse_chat_messages_futures(
483 + conversation, mm_data, mm_uuids = await parse_chat_messages_async(
484 [
485 {
486 "role": "user",
@@ -521,7 +501,7 @@ async def test_parse_chat_messages_empty_image_with_uuid_async(
501 assert conversation == [
502 {"role": "user", "content": "<|image_1|>\nWhat's in the image?"}
503 ]
524 - _assert_mm_data_is_image_input(await mm_future, 1, skipped_image_indices=[0])
504 + _assert_mm_data_is_image_input(mm_data, 1, skipped_image_indices=[0])
505 _assert_mm_uuids(mm_uuids, 1, expected_uuids=[image_uuid])
506
507
@@ -533,7 +513,7 @@ async def test_parse_chat_messages_multiple_images_with_uuids_async(
513 image_uuid1 = "my_uuid_1"
514 image_uuid2 = "my_uuid_2"
515
536 - conversation, mm_future, mm_uuids = parse_chat_messages_futures(
516 + conversation, mm_data, mm_uuids = await parse_chat_messages_async(
517 [
518 {
519 "role": "user",
@@ -562,7 +542,7 @@ async def test_parse_chat_messages_multiple_images_with_uuids_async(
542 "content": "<|image_1|>\n<|image_2|>\nWhat's in these images?",
543 }
544 ]
565 - _assert_mm_data_is_image_input(await mm_future, 2)
545 + _assert_mm_data_is_image_input(mm_data, 2)
546 _assert_mm_uuids(mm_uuids, 2, expected_uuids=[image_uuid1, image_uuid2])
547
548
@@ -574,7 +554,7 @@ async def test_parse_chat_messages_multiple_empty_images_with_uuids_async(
554 image_uuid1 = "my_uuid_1"
555 image_uuid2 = "my_uuid_2"
556
577 - conversation, mm_future, mm_uuids = parse_chat_messages_futures(
557 + conversation, mm_data, mm_uuids = await parse_chat_messages_async(
558 [
559 {
560 "role": "user",
@@ -603,7 +583,7 @@ async def test_parse_chat_messages_multiple_empty_images_with_uuids_async(
583 "content": "<|image_1|>\n<|image_2|>\nWhat's in these images?",
584 }
585 ]
606 - _assert_mm_data_is_image_input(await mm_future, 2, skipped_image_indices=[0, 1])
586 + _assert_mm_data_is_image_input(mm_data, 2, skipped_image_indices=[0, 1])
587 _assert_mm_uuids(mm_uuids, 2, expected_uuids=[image_uuid1, image_uuid2])
588
589
@@ -614,7 +594,7 @@ async def test_parse_chat_messages_multiple_images_with_partial_uuids_async(
594 ):
595 image_uuid2 = "my_uuid_2"
596
617 - conversation, mm_future, mm_uuids = parse_chat_messages_futures(
597 + conversation, mm_data, mm_uuids = await parse_chat_messages_async(
598 [
599 {
600 "role": "user",
@@ -642,7 +622,7 @@ async def test_parse_chat_messages_multiple_images_with_partial_uuids_async(
622 "content": "<|image_1|>\n<|image_2|>\nWhat's in these images?",
623 }
624 ]
645 - _assert_mm_data_is_image_input(await mm_future, 2)
625 + _assert_mm_data_is_image_input(mm_data, 2)
626 _assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, image_uuid2])
627
628
@@ -689,7 +669,7 @@ async def test_parse_chat_messages_single_image_async(
669 phi3v_model_config,
670 image_url,
671 ):
692 - conversation, mm_future, mm_uuids = parse_chat_messages_futures(
672 + conversation, mm_data, mm_uuids = await parse_chat_messages_async(
673 [
674 {
675 "role": "user",
@@ -706,7 +686,7 @@ async def test_parse_chat_messages_single_image_async(
686 assert conversation == [
687 {"role": "user", "content": "<|image_1|>\nWhat's in the image?"}
688 ]
709 - _assert_mm_data_is_image_input(await mm_future, 1)
689 + _assert_mm_data_is_image_input(mm_data, 1)
690 _assert_mm_uuids(mm_uuids, 1, expected_uuids=[None])
691
692
@@ -890,7 +870,7 @@ async def test_parse_chat_messages_audio_embeds_async(
870 # Encode it as base64
871 base64_audio_embedding = tensor2base64(audio_embedding)
872
893 - conversation, mm_future, mm_uuids = parse_chat_messages_futures(
873 + conversation, mm_data, mm_uuids = await parse_chat_messages_async(
874 [
875 {
876 "role": "user",
@@ -908,7 +888,6 @@ async def test_parse_chat_messages_audio_embeds_async(
888 )
889
890 # Should have audio embedding in mm_data (single tensor, not a list)
911 - mm_data = await mm_future
891 assert mm_data is not None
892 assert "audio" in mm_data
893 assert isinstance(mm_data["audio"], torch.Tensor)
@@ -1050,7 +1029,7 @@ async def test_parse_chat_messages_multiple_image_embeds_async(
1029 base64_image_embedding_1 = tensor2base64(image_embedding_1)
1030 base64_image_embedding_2 = tensor2base64(image_embedding_2)
1031
1053 - conversation, mm_future, mm_uuids = parse_chat_messages_futures(
1032 + conversation, mm_data, mm_uuids = await parse_chat_messages_async(
1033 [
1034 {
1035 "role": "user",
@@ -1080,7 +1059,6 @@ async def test_parse_chat_messages_multiple_image_embeds_async(
1059 ]
1060
1061 # Await the future and verify mm_data
1083 - mm_data = await mm_future
1062 assert mm_data is not None
1063 assert "image" in mm_data
1064 assert isinstance(mm_data["image"], list)
@@ -1101,7 +1079,7 @@ async def test_parse_chat_messages_empty_image_embeds_with_uuid_async(
1079 phi3v_model_config_image_embeds,
1080 ):
1081 uuid = "abcd"
1104 - conversation, mm_future, mm_uuids = parse_chat_messages_futures(
1082 + conversation, mm_data, mm_uuids = await parse_chat_messages_async(
1083 [
1084 {
1085 "role": "user",
@@ -1121,7 +1099,6 @@ async def test_parse_chat_messages_empty_image_embeds_with_uuid_async(
1099 "content": "<|image_1|>\nWhat's in this image?",
1100 }
1101 ]
1124 - mm_data = await mm_future
1102 assert mm_data is not None
1103 assert "image" in mm_data
1104 assert isinstance(mm_data["image"], list)
@@ -1228,7 +1205,7 @@ async def test_parse_chat_messages_multiple_images_async(
1205 phi3v_model_config,
1206 image_url,
1207 ):
1231 - conversation, mm_future, mm_uuids = parse_chat_messages_futures(
1208 + conversation, mm_data, mm_uuids = await parse_chat_messages_async(
1209 [
1210 {
1211 "role": "user",
@@ -1252,7 +1229,7 @@ async def test_parse_chat_messages_multiple_images_async(
1229 "content": "<|image_1|>\n<|image_2|>\nWhat's in these images?",
1230 }
1231 ]
1255 - _assert_mm_data_is_image_input(await mm_future, 2)
1232 + _assert_mm_data_is_image_input(mm_data, 2)
1233 _assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None])
1234
1235
@@ -1582,7 +1559,7 @@ async def test_parse_chat_messages_multiple_images_interleave_async(
1559 phi3v_model_config_mm_interleaved,
1560 image_url,
1561 ):
1585 - conversation, mm_data, mm_uuids = parse_chat_messages_futures(
1562 + conversation, mm_data, mm_uuids = await parse_chat_messages_async(
1563 [
1564 {
1565 "role": "user",
@@ -1609,7 +1586,7 @@ async def test_parse_chat_messages_multiple_images_interleave_async(
1586 "Do they have differences?",
1587 }
1588 ]
1612 - _assert_mm_data_is_image_input(await mm_data, 2)
1589 + _assert_mm_data_is_image_input(mm_data, 2)
1590 _assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None])
1591
1592
@@ -1619,7 +1596,7 @@ async def test_parse_chat_messages_multiple_images_with_uuids_interleave_async(
1596 image_url,
1597 ):
1598 image_uuid = str(hash(image_url))
1622 - conversation, mm_data, mm_uuids = parse_chat_messages_futures(
1599 + conversation, mm_data, mm_uuids = await parse_chat_messages_async(
1600 [
1601 {
1602 "role": "user",
@@ -1654,7 +1631,7 @@ async def test_parse_chat_messages_multiple_images_with_uuids_interleave_async(
1631 "Do they have differences?",
1632 }
1633 ]
1657 - _assert_mm_data_is_image_input(await mm_data, 2)
1634 + _assert_mm_data_is_image_input(mm_data, 2)
1635 _assert_mm_uuids(mm_uuids, 2, expected_uuids=[image_uuid, image_uuid])
1636
1637
@@ -2030,377 +2007,6 @@ def test_parse_chat_messages_multiple_images_interleave_with_placeholders(
2007 )
2008
2009
2033 -@pytest.mark.parametrize(
2034 - "model",
2035 - [
2036 - QWEN2VL_MODEL_ID, # tokenizer.chat_template is of type str
2037 - HERMES_MODEL_ID, # tokenizer.chat_template is of type dict
2038 - ],
2039 -)
2040 -@pytest.mark.parametrize("use_tools", [True, False])
2041 -def test_resolve_hf_chat_template(sample_json_schema, model, use_tools):
2042 - """checks that chat_template is a dict type for HF models."""
2043 - model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
2044 - model_info.check_available_online(on_fail="skip")
2045 -
2046 - model_config = ModelConfig(
2047 - model,
2048 - tokenizer=model_info.tokenizer or model,
2049 - tokenizer_mode=model_info.tokenizer_mode,
2050 - revision=model_info.revision,
2051 - trust_remote_code=model_info.trust_remote_code,
2052 - hf_overrides=model_info.hf_overrides,
2053 - skip_tokenizer_init=model_info.require_embed_inputs,
2054 - enable_prompt_embeds=model_info.require_embed_inputs,
2055 - enable_mm_embeds=model_info.require_embed_inputs,
2056 - enforce_eager=model_info.enforce_eager,
2057 - dtype=model_info.dtype,
2058 - )
2059 -
2060 - # Build the tokenizer
2061 - tokenizer = get_tokenizer(
2062 - model,
2063 - trust_remote_code=model_config.trust_remote_code,
2064 - )
2065 -
2066 - tools = (
2067 - [
2068 - {
2069 - "type": "function",
2070 - "function": {
2071 - "name": "dummy_function_name",
2072 - "description": "This is a dummy function",
2073 - "parameters": sample_json_schema,
2074 - },
2075 - }
2076 - ]
2077 - if use_tools
2078 - else None
2079 - )
2080 -
2081 - # Test detecting the tokenizer's chat_template
2082 - chat_template = resolve_hf_chat_template(
2083 - tokenizer,
2084 - chat_template=None,
2085 - tools=tools,
2086 - model_config=model_config,
2087 - )
2088 - assert isinstance(chat_template, str)
2089 -
2090 -
2091 -@pytest.mark.parametrize(
2092 - "model, expected_kwargs",
2093 - [
2094 - (
2095 - QWEN2VL_MODEL_ID,
2096 - {
2097 - "add_vision_id",
2098 - "add_generation_prompt",
2099 - "continue_final_message",
2100 - "tools",
2101 - },
2102 - ),
2103 - (
2104 - QWEN3_MODEL_ID,
2105 - {
2106 - "enable_thinking",
2107 - "add_generation_prompt",
2108 - "continue_final_message",
2109 - "tools",
2110 - },
2111 - ),
2112 - ],
2113 -)
2114 -def test_resolve_hf_chat_template_kwargs(sample_json_schema, model, expected_kwargs):
2115 - """checks that chat_template is a dict type for HF models."""
2116 - model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
2117 - model_info.check_available_online(on_fail="skip")
2118 -
2119 - tools = [
2120 - {
2121 - "type": "function",
2122 - "function": {
2123 - "name": "dummy_function_name",
2124 - "description": "This is a dummy function",
2125 - "parameters": sample_json_schema,
2126 - },
2127 - }
2128 - ]
2129 -
2130 - chat_template_kwargs = {
2131 - # both unused
2132 - "unsed_kwargs_1": 123,
2133 - "unsed_kwargs_2": "abc",
2134 - # should not appear
2135 - "chat_template": "{% Hello world! %}",
2136 - "tokenize": True,
2137 - # used by tokenizer
2138 - "continue_final_message": True,
2139 - "tools": tools,
2140 - # both used by Qwen2-VL and Qwen3
2141 - "add_generation_prompt": True,
2142 - # only used by Qwen2-VL
2143 - "add_vision_id": True,
2144 - # only used by Qwen3
2145 - "enable_thinking": True,
2146 - }
2147 -
2148 - model_config = ModelConfig(
2149 - model,
2150 - tokenizer=model_info.tokenizer or model,
2151 - tokenizer_mode=model_info.tokenizer_mode,
2152 - revision=model_info.revision,
2153 - trust_remote_code=model_info.trust_remote_code,
2154 - hf_overrides=model_info.hf_overrides,
2155 - skip_tokenizer_init=model_info.require_embed_inputs,
2156 - enable_prompt_embeds=model_info.require_embed_inputs,
2157 - enable_mm_embeds=model_info.require_embed_inputs,
2158 - enforce_eager=model_info.enforce_eager,
2159 - dtype=model_info.dtype,
2160 - )
2161 -
2162 - # Build the tokenizer
2163 - tokenizer = get_tokenizer(
2164 - model,
2165 - trust_remote_code=model_config.trust_remote_code,
2166 - )
2167 -
2168 - # Test detecting the tokenizer's chat_template
2169 - chat_template = resolve_hf_chat_template(
2170 - tokenizer,
2171 - chat_template=None,
2172 - tools=tools,
2173 - model_config=model_config,
2174 - )
2175 - with pytest.raises(
2176 - ValueError, match="Found unexpected chat template kwargs from request"
2177 - ):
2178 - # should raise error if `chat_template_kwargs` contains
2179 - # `chat_template` or `tokenize`
2180 - resolve_chat_template_kwargs(
2181 - tokenizer,
2182 - chat_template=chat_template,
2183 - chat_template_kwargs=chat_template_kwargs,
2184 - )
2185 - resolved_chat_template_kwargs = resolve_chat_template_kwargs(
2186 - tokenizer,
2187 - chat_template=chat_template,
2188 - chat_template_kwargs=chat_template_kwargs,
2189 - raise_on_unexpected=False,
2190 - )
2191 - assert set(resolved_chat_template_kwargs.keys()) == expected_kwargs
2192 -
2193 - # Additional test: Verify HF base parameters work with **kwargs tokenizers
2194 - # This validates the fix for tokenizers like Kimi K2 that use **kwargs
2195 - # to receive standard HuggingFace parameters instead of declaring them explicitly
2196 - from vllm.entrypoints.chat_utils import _get_hf_base_chat_template_params
2197 -
2198 - hf_base_params = _get_hf_base_chat_template_params()
2199 - # Verify common HF parameters are in the base class
2200 - assert {"add_generation_prompt", "tools", "continue_final_message"}.issubset(
2201 - hf_base_params
2202 - ), f"Expected HF base params not found in {hf_base_params}"
2203 -
2204 - # Test with a mock tokenizer that uses **kwargs (like Kimi K2)
2205 - class MockTokenizerWithKwargs:
2206 - def apply_chat_template(self, conversation, **kwargs):
2207 - return "mocked_output"
2208 -
2209 - mock_tokenizer = MockTokenizerWithKwargs()
2210 - mock_kwargs = {
2211 - "add_generation_prompt": True,
2212 - "tools": tools,
2213 - "continue_final_message": False,
2214 - "unknown_param": "should_be_filtered",
2215 - }
2216 - resolved_mock = resolve_chat_template_kwargs(
2217 - mock_tokenizer, chat_template, mock_kwargs, raise_on_unexpected=False
2218 - )
2219 - # HF base params should pass through even with **kwargs tokenizer
2220 - assert "add_generation_prompt" in resolved_mock
2221 - assert "tools" in resolved_mock
2222 - assert "continue_final_message" in resolved_mock
2223 - # Unknown params should be filtered out
2224 - assert "unknown_param" not in resolved_mock
2225 -
2226 -
2227 -# NOTE: Qwen2-Audio default chat template is specially defined inside
2228 -# processor class instead of using `tokenizer_config.json`
2229 -@pytest.mark.parametrize(
2230 - ("model", "expected_format"),
2231 - [
2232 - (PHI3V_MODEL_ID, "string"),
2233 - (QWEN2VL_MODEL_ID, "openai"),
2234 - (QWEN25VL_MODEL_ID, "openai"),
2235 - (ULTRAVOX_MODEL_ID, "string"),
2236 - (QWEN2AUDIO_MODEL_ID, "openai"),
2237 - (LLAMA_GUARD_MODEL_ID, "openai"),
2238 - ],
2239 -)
2240 -def test_resolve_content_format_hf_defined(model, expected_format):
2241 - model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
2242 - model_info.check_available_online(on_fail="skip")
2243 -
2244 - model_config = ModelConfig(
2245 - model,
2246 - tokenizer=model_info.tokenizer or model,
2247 - tokenizer_mode=model_info.tokenizer_mode,
2248 - revision=model_info.revision,
2249 - trust_remote_code=model_info.trust_remote_code,
2250 - hf_overrides=model_info.hf_overrides,
2251 - skip_tokenizer_init=model_info.require_embed_inputs,
2252 - enable_prompt_embeds=model_info.require_embed_inputs,
2253 - enable_mm_embeds=model_info.require_embed_inputs,
2254 - enforce_eager=model_info.enforce_eager,
2255 - dtype=model_info.dtype,
2256 - )
2257 -
2258 - tokenizer = get_tokenizer(
2259 - model,
2260 - trust_remote_code=model_config.trust_remote_code,
2261 - )
2262 -
2263 - # Test detecting the tokenizer's chat_template
2264 - chat_template = resolve_hf_chat_template(
2265 - tokenizer,
2266 - chat_template=None,
2267 - tools=None,
2268 - model_config=model_config,
2269 - )
2270 - assert isinstance(chat_template, str)
2271 -
2272 - print("[TEXT]")
2273 - print(chat_template)
2274 - print("[AST]")
2275 - print(_try_extract_ast(chat_template))
2276 -
2277 - resolved_format = resolve_chat_template_content_format(
2278 - None, # Test detecting the tokenizer's chat_template
2279 - None,
2280 - "auto",
2281 - tokenizer,
2282 - model_config=model_config,
2283 - )
2284 -
2285 - assert resolved_format == expected_format
2286 -
2287 -
2288 -@pytest.mark.parametrize(
2289 - ("model", "expected_format"),
2290 - [
2291 - ("Salesforce/blip2-opt-2.7b", "string"),
2292 - ("facebook/chameleon-7b", "string"),
2293 - ("deepseek-ai/deepseek-vl2-tiny", "string"),
2294 - ("adept/fuyu-8b", "string"),
2295 - ("google/paligemma-3b-mix-224", "string"),
2296 - ("Qwen/Qwen-VL", "string"),
2297 - ("Qwen/Qwen-VL-Chat", "string"),
2298 - ],
2299 -)
2300 -def test_resolve_content_format_fallbacks(model, expected_format):
2301 - model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
2302 - model_info.check_available_online(on_fail="skip")
2303 -
2304 - model_config = ModelConfig(
2305 - model,
2306 - tokenizer=model_info.tokenizer or model,
2307 - tokenizer_mode=model_info.tokenizer_mode,
2308 - revision=model_info.revision,
2309 - trust_remote_code=model_info.trust_remote_code,
2310 - hf_overrides=model_info.hf_overrides,
2311 - skip_tokenizer_init=model_info.require_embed_inputs,
2312 - enable_prompt_embeds=model_info.require_embed_inputs,
2313 - enable_mm_embeds=model_info.require_embed_inputs,
2314 - enforce_eager=model_info.enforce_eager,
2315 - dtype=model_info.dtype,
2316 - )
2317 -
2318 - tokenizer = get_tokenizer(
2319 - model_config.tokenizer,
2320 - trust_remote_code=model_config.trust_remote_code,
2321 - )
2322 -
2323 - # Test detecting the tokenizer's chat_template
2324 - chat_template = resolve_hf_chat_template(
2325 - tokenizer,
2326 - chat_template=None,
2327 - tools=None,
2328 - model_config=model_config,
2329 - )
2330 - assert isinstance(chat_template, str)
2331 -
2332 - print("[TEXT]")
2333 - print(chat_template)
2334 - print("[AST]")
2335 - print(_try_extract_ast(chat_template))
2336 -
2337 - resolved_format = resolve_chat_template_content_format(
2338 - None, # Test detecting the tokenizer's chat_template
2339 - None,
2340 - "auto",
2341 - tokenizer,
2342 - model_config=model_config,
2343 - )
2344 -
2345 - assert resolved_format == expected_format
2346 -
2347 -
2348 -@pytest.mark.parametrize(
2349 - ("template_path", "expected_format"),
2350 - [
2351 - ("template_alpaca.jinja", "string"),
2352 - ("template_baichuan.jinja", "string"),
2353 - ("template_chatglm.jinja", "string"),
2354 - ("template_chatglm2.jinja", "string"),
2355 - ("template_chatml.jinja", "string"),
2356 - ("template_dse_qwen2_vl.jinja", "openai"),
2357 - ("template_falcon_180b.jinja", "string"),
2358 - ("template_falcon.jinja", "string"),
2359 - ("template_inkbot.jinja", "string"),
2360 - ("template_teleflm.jinja", "string"),
2361 - ("template_vlm2vec_phi3v.jinja", "openai"),
2362 - ("template_vlm2vec_qwen2vl.jinja", "openai"),
2363 - ("tool_chat_template_granite_20b_fc.jinja", "string"),
2364 - ("tool_chat_template_hermes.jinja", "string"),
2365 - ("tool_chat_template_internlm2_tool.jinja", "string"),
2366 - ("tool_chat_template_llama3.1_json.jinja", "openai"),
2367 - ("tool_chat_template_llama3.2_json.jinja", "openai"),
2368 - ("tool_chat_template_mistral_parallel.jinja", "string"),
2369 - ("tool_chat_template_mistral.jinja", "string"),
2370 - ],
2371 -)
2372 -def test_resolve_content_format_examples(template_path, expected_format):
2373 - model_config = ModelConfig(
2374 - PHI3V_MODEL_ID, # Dummy
2375 - tokenizer=PHI3V_MODEL_ID, # Dummy
2376 - trust_remote_code=True,
2377 - )
2378 -
2379 - dummy_tokenizer = get_tokenizer(
2380 - PHI3V_MODEL_ID, # Dummy
2381 - trust_remote_code=model_config.trust_remote_code,
2382 - )
2383 - dummy_tokenizer.chat_template = None
2384 -
2385 - chat_template = load_chat_template(EXAMPLES_DIR / template_path)
2386 - assert isinstance(chat_template, str)
2387 -
2388 - print("[TEXT]")
2389 - print(chat_template)
2390 - print("[AST]")
2391 - print(_try_extract_ast(chat_template))
2392 -
2393 - resolved_format = resolve_chat_template_content_format(
2394 - chat_template,
2395 - None,
2396 - "auto",
2397 - dummy_tokenizer,
2398 - model_config=model_config,
2399 - )
2400 -
2401 - assert resolved_format == expected_format
2402 -
2403 -
2010 def test_parse_chat_messages_include_thinking_chunk(mistral_model_config):
2011 messages = [
2012 {
@@ -2462,56 +2068,6 @@ def test_parse_chat_messages_include_thinking_chunk(mistral_model_config):
2068 assert conversation_with_thinking == expected_conversation
2069
2070
2465 -def test_apply_mistral_chat_template_thinking_chunk():
2466 - messages = [
2467 - {
2468 - "role": "system",
2469 - "content": [
2470 - {"type": "text", "text": "You are a helpful assistant."},
2471 - {
2472 - "type": "thinking",
2473 - "closed": True,
2474 - "thinking": "Only return the answer when you are confident.",
2475 - },
2476 - ],
2477 - },
2478 - {"role": "user", "content": "What is 2+2?"},
2479 - {
2480 - "role": "assistant",
2481 - "content": [
2482 - {"type": "text", "text": "Let me think about it."},
2483 - {"type": "thinking", "closed": True, "thinking": "2+2 = 4"},
2484 - {
2485 - "type": "text",
2486 - "text": "The answer is 4.",
2487 - },
2488 - ],
2489 - },
2490 - {"role": "user", "content": "Thanks, what is 3+3?"},
2491 - ]
2492 - mistral_tokenizer = MistralTokenizer.from_pretrained(
2493 - "mistralai/Magistral-Small-2509"
2494 - )
2495 -
2496 - tokens_ids = apply_mistral_chat_template(
2497 - mistral_tokenizer, messages, chat_template=None, tools=None
2498 - )
2499 -
2500 - string_tokens = mistral_tokenizer.mistral.decode(
2501 - tokens_ids, special_token_policy=SpecialTokenPolicy.KEEP
2502 - )
2503 -
2504 - expected_tokens = (
2505 - r"<s>[SYSTEM_PROMPT]You are a helpful assistant.[THINK]Only return the"
2506 - r" answer when you are confident.[/THINK][/SYSTEM_PROMPT]"
2507 - r"[INST]What is 2+2?[/INST]"
2508 - r"Let me think about it.[THINK]2+2 = 4[/THINK]The answer is 4.</s>"
2509 - r"[INST]Thanks, what is 3+3?[/INST]"
2510 - )
2511 -
2512 - assert string_tokens == expected_tokens
2513 -
2514 -
2071 def test_parse_chat_messages_single_empty_audio_with_uuid(
2072 qwen2_audio_model_config,
2073 ):
@@ -2550,7 +2106,7 @@ async def test_parse_chat_messages_single_empty_audio_with_uuid_async(
2106 qwen2_audio_model_config,
2107 ):
2108 audio_uuid = "abcd"
2553 - conversation, mm_future, mm_uuids = parse_chat_messages_futures(
2109 + conversation, mm_data, mm_uuids = await parse_chat_messages_async(
2110 [
2111 {
2112 "role": "user",
@@ -2575,5 +2131,5 @@ async def test_parse_chat_messages_single_empty_audio_with_uuid_async(
2131 "audio say?",
2132 }
2133 ]
2578 - _assert_mm_data_inputs(await mm_future, {"audio": 1})
2134 + _assert_mm_data_inputs(mm_data, {"audio": 1})
2135 _assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=[audio_uuid])
tests/renderers/__init__.py
tests/renderers/test_hf.py new
+537
@@ -0,0 +1,537 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import pytest
5 +
6 +from vllm.config import ModelConfig
7 +from vllm.entrypoints.chat_utils import load_chat_template
8 +from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
9 +from vllm.renderers.hf import (
10 + _get_hf_base_chat_template_params,
11 + _try_extract_ast,
12 + resolve_chat_template,
13 + resolve_chat_template_content_format,
14 + resolve_chat_template_kwargs,
15 + safe_apply_chat_template,
16 +)
17 +from vllm.tokenizers import get_tokenizer
18 +
19 +from ..models.registry import HF_EXAMPLE_MODELS
20 +from ..utils import VLLM_PATH
21 +
22 +EXAMPLES_DIR = VLLM_PATH / "examples"
23 +
24 +chatml_jinja_path = VLLM_PATH / "examples/template_chatml.jinja"
25 +assert chatml_jinja_path.exists()
26 +
27 +# Define models, templates, and their corresponding expected outputs
28 +MODEL_TEMPLATE_GENERATION_OUTPUT = [
29 + (
30 + "facebook/opt-125m",
31 + chatml_jinja_path,
32 + True,
33 + False,
34 + """<|im_start|>user
35 +Hello<|im_end|>
36 +<|im_start|>assistant
37 +Hi there!<|im_end|>
38 +<|im_start|>user
39 +What is the capital of<|im_end|>
40 +<|im_start|>assistant
41 +""",
42 + ),
43 + (
44 + "facebook/opt-125m",
45 + chatml_jinja_path,
46 + False,
47 + False,
48 + """<|im_start|>user
49 +Hello<|im_end|>
50 +<|im_start|>assistant
51 +Hi there!<|im_end|>
52 +<|im_start|>user
53 +What is the capital of""",
54 + ),
55 + (
56 + "facebook/opt-125m",
57 + chatml_jinja_path,
58 + False,
59 + True,
60 + """<|im_start|>user
61 +Hello<|im_end|>
62 +<|im_start|>assistant
63 +Hi there!<|im_end|>
64 +<|im_start|>user
65 +What is the capital of<|im_end|>
66 +<|im_start|>assistant
67 +The capital of""",
68 + ),
69 +]
70 +
71 +TEST_MESSAGES = [
72 + {"role": "user", "content": "Hello"},
73 + {"role": "assistant", "content": "Hi there!"},
74 + {"role": "user", "content": "What is the capital of"},
75 +]
76 +ASSISTANT_MESSAGE_TO_CONTINUE = {"role": "assistant", "content": "The capital of"}
77 +
78 +
79 +def test_load_chat_template():
80 + # Testing chatml template
81 + template_content = load_chat_template(chat_template=chatml_jinja_path)
82 +
83 + # Test assertions
84 + assert template_content is not None
85 + # Hard coded value for template_chatml.jinja
86 + assert (
87 + template_content
88 + == """{% for message in messages %}{{'<|im_start|>' + message['role'] + '\\n' + message['content']}}{% if (loop.last and add_generation_prompt) or not loop.last %}{{ '<|im_end|>' + '\\n'}}{% endif %}{% endfor %}
89 +{% if add_generation_prompt and messages[-1]['role'] != 'assistant' %}{{ '<|im_start|>assistant\\n' }}{% endif %}""" # noqa: E501
90 + )
91 +
92 +
93 +def test_no_load_chat_template_filelike():
94 + # Testing chatml template
95 + template = "../../examples/does_not_exist"
96 +
97 + with pytest.raises(ValueError, match="looks like a file path"):
98 + load_chat_template(chat_template=template)
99 +
100 +
101 +def test_no_load_chat_template_literallike():
102 + # Testing chatml template
103 + template = "{{ messages }}"
104 +
105 + template_content = load_chat_template(chat_template=template)
106 +
107 + assert template_content == template
108 +
109 +
110 +@pytest.mark.parametrize(
111 + "model",
112 + [
113 + "Qwen/Qwen2-VL-2B-Instruct", # chat_template is of type str
114 + "NousResearch/Hermes-3-Llama-3.1-8B", # chat_template is of type dict
115 + ],
116 +)
117 +@pytest.mark.parametrize("use_tools", [True, False])
118 +def test_resolve_chat_template(sample_json_schema, model, use_tools):
119 + """checks that chat_template is a dict type for HF models."""
120 + model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
121 + model_info.check_available_online(on_fail="skip")
122 +
123 + model_config = ModelConfig(
124 + model,
125 + tokenizer=model_info.tokenizer or model,
126 + tokenizer_mode=model_info.tokenizer_mode,
127 + revision=model_info.revision,
128 + trust_remote_code=model_info.trust_remote_code,
129 + hf_overrides=model_info.hf_overrides,
130 + skip_tokenizer_init=model_info.require_embed_inputs,
131 + enable_prompt_embeds=model_info.require_embed_inputs,
132 + enable_mm_embeds=model_info.require_embed_inputs,
133 + enforce_eager=model_info.enforce_eager,
134 + dtype=model_info.dtype,
135 + )
136 +
137 + # Build the tokenizer
138 + tokenizer = get_tokenizer(
139 + model,
140 + trust_remote_code=model_config.trust_remote_code,
141 + )
142 +
143 + tools = (
144 + [
145 + {
146 + "type": "function",
147 + "function": {
148 + "name": "dummy_function_name",
149 + "description": "This is a dummy function",
150 + "parameters": sample_json_schema,
151 + },
152 + }
153 + ]
154 + if use_tools
155 + else None
156 + )
157 +
158 + # Test detecting the tokenizer's chat_template
159 + chat_template = resolve_chat_template(
160 + tokenizer,
161 + chat_template=None,
162 + tools=tools,
163 + model_config=model_config,
164 + )
165 + assert isinstance(chat_template, str)
166 +
167 +
168 +@pytest.mark.parametrize(
169 + "model, expected_kwargs",
170 + [
171 + (
172 + "Qwen/Qwen2-VL-2B-Instruct",
173 + {
174 + "add_vision_id",
175 + "add_generation_prompt",
176 + "continue_final_message",
177 + "tools",
178 + },
179 + ),
180 + (
181 + "Qwen/Qwen3-8B",
182 + {
183 + "enable_thinking",
184 + "add_generation_prompt",
185 + "continue_final_message",
186 + "tools",
187 + },
188 + ),
189 + ],
190 +)
191 +def test_resolve_chat_template_kwargs(sample_json_schema, model, expected_kwargs):
192 + """checks that chat_template is a dict type for HF models."""
193 + model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
194 + model_info.check_available_online(on_fail="skip")
195 +
196 + tools = [
197 + {
198 + "type": "function",
199 + "function": {
200 + "name": "dummy_function_name",
201 + "description": "This is a dummy function",
202 + "parameters": sample_json_schema,
203 + },
204 + }
205 + ]
206 +
207 + chat_template_kwargs = {
208 + # both unused
209 + "unsed_kwargs_1": 123,
210 + "unsed_kwargs_2": "abc",
211 + # should not appear
212 + "chat_template": "{% Hello world! %}",
213 + "tokenize": True,
214 + # used by tokenizer
215 + "continue_final_message": True,
216 + "tools": tools,
217 + # both used by Qwen2-VL and Qwen3
218 + "add_generation_prompt": True,
219 + # only used by Qwen2-VL
220 + "add_vision_id": True,
221 + # only used by Qwen3
222 + "enable_thinking": True,
223 + }
224 +
225 + model_config = ModelConfig(
226 + model,
227 + tokenizer=model_info.tokenizer or model,
228 + tokenizer_mode=model_info.tokenizer_mode,
229 + revision=model_info.revision,
230 + trust_remote_code=model_info.trust_remote_code,
231 + hf_overrides=model_info.hf_overrides,
232 + skip_tokenizer_init=model_info.require_embed_inputs,
233 + enable_prompt_embeds=model_info.require_embed_inputs,
234 + enable_mm_embeds=model_info.require_embed_inputs,
235 + enforce_eager=model_info.enforce_eager,
236 + dtype=model_info.dtype,
237 + )
238 +
239 + # Build the tokenizer
240 + tokenizer = get_tokenizer(
241 + model,
242 + trust_remote_code=model_config.trust_remote_code,
243 + )
244 +
245 + # Test detecting the tokenizer's chat_template
246 + chat_template = resolve_chat_template(
247 + tokenizer,
248 + chat_template=None,
249 + tools=tools,
250 + model_config=model_config,
251 + )
252 + with pytest.raises(
253 + ValueError, match="Found unexpected chat template kwargs from request"
254 + ):
255 + # should raise error if `chat_template_kwargs` contains
256 + # `chat_template` or `tokenize`
257 + resolve_chat_template_kwargs(
258 + tokenizer,
259 + chat_template=chat_template,
260 + chat_template_kwargs=chat_template_kwargs,
261 + )
262 + resolved_chat_template_kwargs = resolve_chat_template_kwargs(
263 + tokenizer,
264 + chat_template=chat_template,
265 + chat_template_kwargs=chat_template_kwargs,
266 + raise_on_unexpected=False,
267 + )
268 + assert set(resolved_chat_template_kwargs.keys()) == expected_kwargs
269 +
270 + # Additional test: Verify HF base parameters work with **kwargs tokenizers
271 + # This validates the fix for tokenizers like Kimi K2 that use **kwargs
272 + # to receive standard HuggingFace parameters instead of declaring them explicitly
273 + hf_base_params = _get_hf_base_chat_template_params()
274 + # Verify common HF parameters are in the base class
275 + assert {"add_generation_prompt", "tools", "continue_final_message"}.issubset(
276 + hf_base_params
277 + ), f"Expected HF base params not found in {hf_base_params}"
278 +
279 + # Test with a mock tokenizer that uses **kwargs (like Kimi K2)
280 + class MockTokenizerWithKwargs:
281 + def apply_chat_template(self, conversation, **kwargs):
282 + return "mocked_output"
283 +
284 + mock_tokenizer = MockTokenizerWithKwargs()
285 + mock_kwargs = {
286 + "add_generation_prompt": True,
287 + "tools": tools,
288 + "continue_final_message": False,
289 + "unknown_param": "should_be_filtered",
290 + }
291 + resolved_mock = resolve_chat_template_kwargs(
292 + mock_tokenizer, chat_template, mock_kwargs, raise_on_unexpected=False
293 + )
294 + # HF base params should pass through even with **kwargs tokenizer
295 + assert "add_generation_prompt" in resolved_mock
296 + assert "tools" in resolved_mock
297 + assert "continue_final_message" in resolved_mock
298 + # Unknown params should be filtered out
299 + assert "unknown_param" not in resolved_mock
300 +
301 +
302 +# NOTE: Qwen2-Audio default chat template is specially defined inside
303 +# processor class instead of using `tokenizer_config.json`
304 +@pytest.mark.parametrize(
305 + ("model", "expected_format"),
306 + [
307 + ("microsoft/Phi-3.5-vision-instruct", "string"),
308 + ("Qwen/Qwen2-VL-2B-Instruct", "openai"),
309 + ("Qwen/Qwen2.5-VL-3B-Instruct", "openai"),
310 + ("fixie-ai/ultravox-v0_5-llama-3_2-1b", "string"),
311 + ("Qwen/Qwen2-Audio-7B-Instruct", "openai"),
312 + ("meta-llama/Llama-Guard-3-1B", "openai"),
313 + ],
314 +)
315 +def test_resolve_content_format_hf_defined(model, expected_format):
316 + model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
317 + model_info.check_available_online(on_fail="skip")
318 +
319 + model_config = ModelConfig(
320 + model,
321 + tokenizer=model_info.tokenizer or model,
322 + tokenizer_mode=model_info.tokenizer_mode,
323 + revision=model_info.revision,
324 + trust_remote_code=model_info.trust_remote_code,
325 + hf_overrides=model_info.hf_overrides,
326 + skip_tokenizer_init=model_info.require_embed_inputs,
327 + enable_prompt_embeds=model_info.require_embed_inputs,
328 + enable_mm_embeds=model_info.require_embed_inputs,
329 + enforce_eager=model_info.enforce_eager,
330 + dtype=model_info.dtype,
331 + )
332 +
333 + tokenizer = get_tokenizer(
334 + model,
335 + trust_remote_code=model_config.trust_remote_code,
336 + )
337 +
338 + # Test detecting the tokenizer's chat_template
339 + chat_template = resolve_chat_template(
340 + tokenizer,
341 + chat_template=None,
342 + tools=None,
343 + model_config=model_config,
344 + )
345 + assert isinstance(chat_template, str)
346 +
347 + print("[TEXT]")
348 + print(chat_template)
349 + print("[AST]")
350 + print(_try_extract_ast(chat_template))
351 +
352 + resolved_format = resolve_chat_template_content_format(
353 + None, # Test detecting the tokenizer's chat_template
354 + None,
355 + "auto",
356 + tokenizer,
357 + model_config=model_config,
358 + )
359 +
360 + assert resolved_format == expected_format
361 +
362 +
363 +@pytest.mark.parametrize(
364 + ("model", "expected_format"),
365 + [
366 + ("Salesforce/blip2-opt-2.7b", "string"),
367 + ("facebook/chameleon-7b", "string"),
368 + ("deepseek-ai/deepseek-vl2-tiny", "string"),
369 + ("adept/fuyu-8b", "string"),
370 + ("google/paligemma-3b-mix-224", "string"),
371 + ("Qwen/Qwen-VL", "string"),
372 + ("Qwen/Qwen-VL-Chat", "string"),
373 + ],
374 +)
375 +def test_resolve_content_format_fallbacks(model, expected_format):
376 + model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
377 + model_info.check_available_online(on_fail="skip")
378 +
379 + model_config = ModelConfig(
380 + model,
381 + tokenizer=model_info.tokenizer or model,
382 + tokenizer_mode=model_info.tokenizer_mode,
383 + revision=model_info.revision,
384 + trust_remote_code=model_info.trust_remote_code,
385 + hf_overrides=model_info.hf_overrides,
386 + skip_tokenizer_init=model_info.require_embed_inputs,
387 + enable_prompt_embeds=model_info.require_embed_inputs,
388 + enable_mm_embeds=model_info.require_embed_inputs,
389 + enforce_eager=model_info.enforce_eager,
390 + dtype=model_info.dtype,
391 + )
392 +
393 + tokenizer = get_tokenizer(
394 + model_config.tokenizer,
395 + trust_remote_code=model_config.trust_remote_code,
396 + )
397 +
398 + # Test detecting the tokenizer's chat_template
399 + chat_template = resolve_chat_template(
400 + tokenizer,
401 + chat_template=None,
402 + tools=None,
403 + model_config=model_config,
404 + )
405 + assert isinstance(chat_template, str)
406 +
407 + print("[TEXT]")
408 + print(chat_template)
409 + print("[AST]")
410 + print(_try_extract_ast(chat_template))
411 +
412 + resolved_format = resolve_chat_template_content_format(
413 + None, # Test detecting the tokenizer's chat_template
414 + None,
415 + "auto",
416 + tokenizer,
417 + model_config=model_config,
418 + )
419 +
420 + assert resolved_format == expected_format
421 +
422 +
423 +@pytest.mark.parametrize(
424 + ("template_path", "expected_format"),
425 + [
426 + ("template_alpaca.jinja", "string"),
427 + ("template_baichuan.jinja", "string"),
428 + ("template_chatglm.jinja", "string"),
429 + ("template_chatglm2.jinja", "string"),
430 + ("template_chatml.jinja", "string"),
431 + ("template_dse_qwen2_vl.jinja", "openai"),
432 + ("template_falcon_180b.jinja", "string"),
433 + ("template_falcon.jinja", "string"),
434 + ("template_inkbot.jinja", "string"),
435 + ("template_teleflm.jinja", "string"),
436 + ("template_vlm2vec_phi3v.jinja", "openai"),
437 + ("template_vlm2vec_qwen2vl.jinja", "openai"),
438 + ("tool_chat_template_granite_20b_fc.jinja", "string"),
439 + ("tool_chat_template_hermes.jinja", "string"),
440 + ("tool_chat_template_internlm2_tool.jinja", "string"),
441 + ("tool_chat_template_llama3.1_json.jinja", "openai"),
442 + ("tool_chat_template_llama3.2_json.jinja", "openai"),
443 + ("tool_chat_template_mistral_parallel.jinja", "string"),
444 + ("tool_chat_template_mistral.jinja", "string"),
445 + ],
446 +)
447 +def test_resolve_content_format_examples(template_path, expected_format):
448 + model = "Qwen/Qwen2-VL-2B-Instruct" # Dummy
449 + model_config = ModelConfig(
450 + model,
451 + tokenizer=model,
452 + trust_remote_code=True,
453 + )
454 +
455 + dummy_tokenizer = get_tokenizer(
456 + model,
457 + trust_remote_code=model_config.trust_remote_code,
458 + )
459 + dummy_tokenizer.chat_template = None
460 +
461 + chat_template = load_chat_template(EXAMPLES_DIR / template_path)
462 + assert isinstance(chat_template, str)
463 +
464 + print("[TEXT]")
465 + print(chat_template)
466 + print("[AST]")
467 + print(_try_extract_ast(chat_template))
468 +
469 + resolved_format = resolve_chat_template_content_format(
470 + chat_template,
471 + None,
472 + "auto",
473 + dummy_tokenizer,
474 + model_config=model_config,
475 + )
476 +
477 + assert resolved_format == expected_format
478 +
479 +
480 +@pytest.mark.parametrize(
481 + "model,template,add_generation_prompt,continue_final_message,expected_output",
482 + MODEL_TEMPLATE_GENERATION_OUTPUT,
483 +)
484 +def test_get_gen_prompt(
485 + model, template, add_generation_prompt, continue_final_message, expected_output
486 +):
487 + model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
488 + model_info.check_available_online(on_fail="skip")
489 +
490 + model_config = ModelConfig(
491 + model,
492 + tokenizer=model_info.tokenizer or model,
493 + tokenizer_mode=model_info.tokenizer_mode,
494 + trust_remote_code=model_info.trust_remote_code,
495 + revision=model_info.revision,
496 + hf_overrides=model_info.hf_overrides,
497 + skip_tokenizer_init=model_info.require_embed_inputs,
498 + enable_prompt_embeds=model_info.require_embed_inputs,
499 + enable_mm_embeds=model_info.require_embed_inputs,
500 + enforce_eager=model_info.enforce_eager,
501 + dtype=model_info.dtype,
502 + )
503 +
504 + # Initialize the tokenizer
505 + tokenizer = get_tokenizer(
506 + tokenizer_name=model_config.tokenizer,
507 + trust_remote_code=model_config.trust_remote_code,
508 + )
509 + template_content = load_chat_template(chat_template=template)
510 +
511 + # Create a mock request object using keyword arguments
512 + mock_request = ChatCompletionRequest(
513 + model=model,
514 + messages=TEST_MESSAGES + [ASSISTANT_MESSAGE_TO_CONTINUE]
515 + if continue_final_message
516 + else TEST_MESSAGES,
517 + add_generation_prompt=add_generation_prompt,
518 + continue_final_message=continue_final_message,
519 + )
520 +
521 + # Call the function and get the result
522 + result = safe_apply_chat_template(
523 + model_config,
524 + tokenizer,
525 + mock_request.messages,
526 + tools=None,
527 + chat_template=mock_request.chat_template or template_content,
528 + add_generation_prompt=mock_request.add_generation_prompt,
529 + continue_final_message=mock_request.continue_final_message,
530 + tokenize=False,
531 + )
532 +
533 + # Test assertion
534 + assert result == expected_output, (
535 + f"The generated prompt does not match the expected output for "
536 + f"model {model} and template {template}"
537 + )
tests/renderers/test_mistral.py new
+100
@@ -0,0 +1,100 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import asyncio
5 +import time
6 +from unittest.mock import Mock
7 +
8 +import pytest
9 +from mistral_common.tokens.tokenizers.base import SpecialTokenPolicy
10 +
11 +from vllm.config import ModelConfig
12 +from vllm.renderers.mistral import MistralRenderer, safe_apply_chat_template
13 +from vllm.tokenizers.mistral import MistralTokenizer
14 +
15 +
16 +@pytest.mark.asyncio
17 +async def test_async_mistral_tokenizer_does_not_block_event_loop():
18 + expected_tokens = [1, 2, 3]
19 +
20 + # Mock the blocking version to sleep
21 + def mocked_apply_chat_template(*_args, **_kwargs):
22 + time.sleep(2)
23 + return expected_tokens
24 +
25 + mock_tokenizer = Mock(spec=MistralTokenizer)
26 + mock_tokenizer.apply_chat_template = mocked_apply_chat_template
27 + mock_renderer = MistralRenderer(Mock(spec=ModelConfig), tokenizer_kwargs={})
28 + mock_renderer._tokenizer = mock_tokenizer
29 +
30 + task = mock_renderer.render_messages_async([])
31 +
32 + # Ensure the event loop is not blocked
33 + blocked_count = 0
34 + for _i in range(20): # Check over ~2 seconds
35 + start = time.perf_counter()
36 + await asyncio.sleep(0)
37 + elapsed = time.perf_counter() - start
38 +
39 + # an overly generous elapsed time for slow machines
40 + if elapsed >= 0.5:
41 + blocked_count += 1
42 +
43 + await asyncio.sleep(0.1)
44 +
45 + # Ensure task completes
46 + _, prompt = await task
47 + assert prompt["prompt_token_ids"] == expected_tokens, (
48 + "Mocked blocking tokenizer was not called"
49 + )
50 + assert blocked_count == 0, "Event loop blocked during tokenization"
51 +
52 +
53 +def test_apply_mistral_chat_template_thinking_chunk():
54 + messages = [
55 + {
56 + "role": "system",
57 + "content": [
58 + {"type": "text", "text": "You are a helpful assistant."},
59 + {
60 + "type": "thinking",
61 + "closed": True,
62 + "thinking": "Only return the answer when you are confident.",
63 + },
64 + ],
65 + },
66 + {"role": "user", "content": "What is 2+2?"},
67 + {
68 + "role": "assistant",
69 + "content": [
70 + {"type": "text", "text": "Let me think about it."},
71 + {"type": "thinking", "closed": True, "thinking": "2+2 = 4"},
72 + {
73 + "type": "text",
74 + "text": "The answer is 4.",
75 + },
76 + ],
77 + },
78 + {"role": "user", "content": "Thanks, what is 3+3?"},
79 + ]
80 + mistral_tokenizer = MistralTokenizer.from_pretrained(
81 + "mistralai/Magistral-Small-2509"
82 + )
83 +
84 + tokens_ids = safe_apply_chat_template(
85 + mistral_tokenizer, messages, chat_template=None, tools=None
86 + )
87 +
88 + string_tokens = mistral_tokenizer.mistral.decode(
89 + tokens_ids, special_token_policy=SpecialTokenPolicy.KEEP
90 + )
91 +
92 + expected_tokens = (
93 + r"<s>[SYSTEM_PROMPT]You are a helpful assistant.[THINK]Only return the"
94 + r" answer when you are confident.[/THINK][/SYSTEM_PROMPT]"
95 + r"[INST]What is 2+2?[/INST]"
96 + r"Let me think about it.[THINK]2+2 = 4[/THINK]The answer is 4.</s>"
97 + r"[INST]Thanks, what is 3+3?[/INST]"
98 + )
99 +
100 + assert string_tokens == expected_tokens
tests/test_inputs.py
+2 -3
@@ -7,7 +7,6 @@ from vllm.config import ModelConfig
7 from vllm.inputs import zip_enc_dec_prompts
8 from vllm.inputs.parse import parse_raw_prompts
9 from vllm.inputs.preprocess import InputPreprocessor
10 -from vllm.tokenizers import cached_tokenizer_from_config
10
11 pytestmark = pytest.mark.cpu_test
12
@@ -115,10 +114,10 @@ def test_zip_enc_dec_prompts(mm_processor_kwargs, expected_mm_kwargs):
114 )
115 def test_preprocessor_always_mm_code_path(model_id, prompt):
116 model_config = ModelConfig(model=model_id)
118 - tokenizer = cached_tokenizer_from_config(model_config)
119 - input_preprocessor = InputPreprocessor(model_config, tokenizer)
117 + input_preprocessor = InputPreprocessor(model_config)
118
119 # HF processor adds sep token
120 + tokenizer = input_preprocessor.get_tokenizer()
121 sep_token_id = tokenizer.vocab[tokenizer.sep_token]
122
123 processed_inputs = input_preprocessor.preprocess(prompt)
tests/v1/engine/test_llm_engine.py
+1 -1
@@ -224,7 +224,7 @@ def test_skip_tokenizer_initialization(model: str):
224 )
225 sampling_params = SamplingParams(prompt_logprobs=True, detokenize=True)
226
227 - with pytest.raises(ValueError, match="cannot pass text prompts when"):
227 + with pytest.raises(ValueError, match="`skip_tokenizer_init=True`"):
228 llm.generate("abc", sampling_params)
229
230 outputs = llm.generate(
tests/v1/engine/test_process_multi_modal_uuids.py
+11 -10
@@ -5,7 +5,13 @@ import pytest
5
6 from vllm.assets.image import ImageAsset
7 from vllm.assets.video import VideoAsset
8 -from vllm.config import CacheConfig, DeviceConfig, ModelConfig, VllmConfig
8 +from vllm.config import (
9 + CacheConfig,
10 + DeviceConfig,
11 + ModelConfig,
12 + MultiModalConfig,
13 + VllmConfig,
14 +)
15 from vllm.multimodal import MultiModalRegistry, MultiModalUUIDDict
16 from vllm.sampling_params import SamplingParams
17 from vllm.v1.engine.input_processor import InputProcessor
@@ -44,27 +50,22 @@ def _mock_input_processor(
50 monkeypatch.setattr(VllmConfig, "__post_init__", lambda self: None, raising=True)
51
52 model_config = ModelConfig(
53 + tokenizer="dummy",
54 skip_tokenizer_init=True,
55 max_model_len=128,
56 mm_processor_cache_gb=mm_cache_gb,
57 generation_config="vllm",
51 - tokenizer="dummy",
58 )
59 + model_config.runner_type = "generate"
60 + model_config.multimodal_config = MultiModalConfig(mm_processor_cache_gb=mm_cache_gb)
61
54 - # Minimal multimodal_config to satisfy references in
55 - # Processor.process_inputs.
56 - class _MockMMConfig:
57 - def __init__(self, gb: float):
58 - self.mm_processor_cache_gb = gb
59 -
60 - model_config.multimodal_config = _MockMMConfig(mm_cache_gb) # type: ignore[attr-defined]
62 vllm_config = VllmConfig(
63 model_config=model_config,
64 cache_config=CacheConfig(enable_prefix_caching=enable_prefix_caching),
65 device_config=DeviceConfig(device="cpu"),
66 )
67
67 - return InputProcessor(vllm_config, tokenizer=None)
68 + return InputProcessor(vllm_config)
69
70
71 def test_multi_modal_uuids_length_mismatch_raises(monkeypatch):
tools/pre_commit/mypy.py
+1
@@ -35,6 +35,7 @@ FILES = [
35 "vllm/multimodal",
36 "vllm/platforms",
37 "vllm/plugins",
38 + "vllm/renderers",
39 "vllm/tokenizers",
40 "vllm/transformers_utils",
41 "vllm/triton_utils",
vllm/engine/protocol.py
+5 -6
@@ -11,9 +11,9 @@ from vllm.lora.request import LoRARequest
11 from vllm.outputs import PoolingRequestOutput, RequestOutput
12 from vllm.plugins.io_processors import IOProcessor
13 from vllm.pooling_params import PoolingParams
14 +from vllm.renderers import RendererLike
15 from vllm.sampling_params import SamplingParams
16 from vllm.tasks import SupportedTask
16 -from vllm.tokenizers import TokenizerLike
17 from vllm.v1.engine import EngineCoreRequest
18 from vllm.v1.engine.input_processor import InputProcessor
19
@@ -26,6 +26,10 @@ class EngineClient(ABC):
26 input_processor: InputProcessor
27 io_processor: IOProcessor | None
28
29 + @property
30 + @abstractmethod
31 + def renderer(self) -> RendererLike: ...
32 +
33 @property
34 @abstractmethod
35 def is_running(self) -> bool: ...
@@ -88,11 +92,6 @@ class EngineClient(ABC):
92 """
93 ...
94
91 - @abstractmethod
92 - async def get_tokenizer(self) -> TokenizerLike:
93 - """Get the tokenizer"""
94 - ...
95 -
95 @abstractmethod
96 async def is_tracing_enabled(self) -> bool: ...
97
vllm/entrypoints/chat_utils.py
+27 -508
@@ -2,22 +2,15 @@
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 import asyncio
5 -import inspect
5 import json
6 +import warnings
7 from abc import ABC, abstractmethod
8 -from collections import Counter, defaultdict, deque
8 +from collections import Counter, defaultdict
9 from collections.abc import Awaitable, Callable, Iterable
10 from functools import cached_property, lru_cache, partial
11 from pathlib import Path
12 from typing import TYPE_CHECKING, Any, Generic, Literal, TypeAlias, TypeVar, cast
13
14 -import jinja2
15 -import jinja2.ext
16 -import jinja2.meta
17 -import jinja2.nodes
18 -import jinja2.parser
19 -import jinja2.sandbox
20 -import transformers.utils.chat_template_utils as hf_chat_utils
14 from openai.types.chat import (
15 ChatCompletionAssistantMessageParam,
16 ChatCompletionContentPartImageParam,
@@ -39,7 +32,6 @@ from openai.types.responses import ResponseInputImageParam
32 from openai_harmony import Message as OpenAIHarmonyMessage
33 from PIL import Image
34 from pydantic import BaseModel, ConfigDict, TypeAdapter
42 -from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast, ProcessorMixin
35
36 # pydantic needs the TypedDict from typing_extensions
37 from typing_extensions import Required, TypedDict
@@ -50,24 +42,35 @@ from vllm.logger import init_logger
42 from vllm.model_executor.models import SupportsMultiModal
43 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalDataDict, MultiModalUUIDDict
44 from vllm.multimodal.utils import MEDIA_CONNECTOR_REGISTRY, MediaConnector
53 -from vllm.tokenizers import TokenizerLike
54 -from vllm.transformers_utils.chat_templates import get_chat_template_fallback_path
55 -from vllm.transformers_utils.processor import cached_get_processor
45 from vllm.utils import random_uuid
46 from vllm.utils.collection_utils import is_list_of
58 -from vllm.utils.func_utils import supports_kw
47 from vllm.utils.import_utils import LazyLoader
48
49 if TYPE_CHECKING:
50 import torch
63 -
64 - from vllm.tokenizers.mistral import MistralTokenizer
51 else:
52 torch = LazyLoader("torch", globals(), "torch")
53
54 logger = init_logger(__name__)
55
56
57 +def __getattr__(name: str):
58 + if name == "resolve_hf_chat_template":
59 + from vllm.renderers.hf import resolve_chat_template
60 +
61 + warnings.warn(
62 + "`vllm.entrypoints.chat_utils.resolve_hf_chat_template` has been moved to "
63 + "`vllm.renderers.hf.resolve_chat_template`. "
64 + "The old name will be removed in v0.16.",
65 + DeprecationWarning,
66 + stacklevel=2,
67 + )
68 +
69 + return resolve_chat_template
70 +
71 + raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
72 +
73 +
74 class ChatTemplateResolutionError(ValueError):
75 """Raised when chat template resolution fails.
76
@@ -320,325 +323,8 @@ class ConversationMessage(TypedDict, total=False):
323 # Passed in by user
324 ChatTemplateContentFormatOption = Literal["auto", "string", "openai"]
325
323 -# Used internally
324 -_ChatTemplateContentFormat = Literal["string", "openai"]
325 -
326 -
327 -def _is_var_access(node: jinja2.nodes.Node, varname: str) -> bool:
328 - if isinstance(node, jinja2.nodes.Name):
329 - return node.ctx == "load" and node.name == varname
330 -
331 - return False
332 -
333 -
334 -def _is_attr_access(node: jinja2.nodes.Node, varname: str, key: str) -> bool:
335 - if isinstance(node, jinja2.nodes.Getitem):
336 - return (
337 - _is_var_access(node.node, varname)
338 - and isinstance(node.arg, jinja2.nodes.Const)
339 - and node.arg.value == key
340 - )
341 -
342 - if isinstance(node, jinja2.nodes.Getattr):
343 - return _is_var_access(node.node, varname) and node.attr == key
344 -
345 - return False
346 -
347 -
348 -def _is_var_or_elems_access(
349 - node: jinja2.nodes.Node,
350 - varname: str,
351 - key: str | None = None,
352 -) -> bool:
353 - if isinstance(node, jinja2.nodes.Filter):
354 - return node.node is not None and _is_var_or_elems_access(
355 - node.node, varname, key
356 - )
357 - if isinstance(node, jinja2.nodes.Test):
358 - return _is_var_or_elems_access(node.node, varname, key)
359 -
360 - if isinstance(node, jinja2.nodes.Getitem) and isinstance(
361 - node.arg, jinja2.nodes.Slice
362 - ):
363 - return _is_var_or_elems_access(node.node, varname, key)
364 -
365 - return _is_attr_access(node, varname, key) if key else _is_var_access(node, varname)
366 -
367 -
368 -def _iter_nodes_assign_var_or_elems(root: jinja2.nodes.Node, varname: str):
369 - # Global variable that is implicitly defined at the root
370 - yield root, varname
371 -
372 - # Iterative BFS
373 - related_varnames = deque([varname])
374 - while related_varnames:
375 - related_varname = related_varnames.popleft()
376 -
377 - for assign_ast in root.find_all(jinja2.nodes.Assign):
378 - lhs = assign_ast.target
379 - rhs = assign_ast.node
380 -
381 - if _is_var_or_elems_access(rhs, related_varname):
382 - assert isinstance(lhs, jinja2.nodes.Name)
383 - yield assign_ast, lhs.name
384 -
385 - # Avoid infinite looping for self-assignment
386 - if lhs.name != related_varname:
387 - related_varnames.append(lhs.name)
388 -
389 -
390 -# NOTE: The proper way to handle this is to build a CFG so that we can handle
391 -# the scope in which each variable is defined, but that is too complicated
392 -def _iter_nodes_assign_messages_item(root: jinja2.nodes.Node):
393 - messages_varnames = [
394 - varname for _, varname in _iter_nodes_assign_var_or_elems(root, "messages")
395 - ]
396 -
397 - # Search for {%- for message in messages -%} loops
398 - for loop_ast in root.find_all(jinja2.nodes.For):
399 - loop_iter = loop_ast.iter
400 - loop_target = loop_ast.target
401 -
402 - for varname in messages_varnames:
403 - if _is_var_or_elems_access(loop_iter, varname):
404 - assert isinstance(loop_target, jinja2.nodes.Name)
405 - yield loop_ast, loop_target.name
406 - break
407 -
408 -
409 -def _iter_nodes_assign_content_item(root: jinja2.nodes.Node):
410 - message_varnames = [
411 - varname for _, varname in _iter_nodes_assign_messages_item(root)
412 - ]
413 -
414 - # Search for {%- for content in message['content'] -%} loops
415 - for loop_ast in root.find_all(jinja2.nodes.For):
416 - loop_iter = loop_ast.iter
417 - loop_target = loop_ast.target
418 -
419 - for varname in message_varnames:
420 - if _is_var_or_elems_access(loop_iter, varname, "content"):
421 - assert isinstance(loop_target, jinja2.nodes.Name)
422 - yield loop_ast, loop_target.name
423 - break
424 -
425 -
426 -def _try_extract_ast(chat_template: str) -> jinja2.nodes.Template | None:
427 - try:
428 - jinja_compiled = hf_chat_utils._compile_jinja_template(chat_template)
429 - return jinja_compiled.environment.parse(chat_template)
430 - except Exception:
431 - logger.exception("Error when compiling Jinja template")
432 - return None
433 -
434 -
435 -@lru_cache(maxsize=32)
436 -def _detect_content_format(
437 - chat_template: str,
438 - *,
439 - default: _ChatTemplateContentFormat,
440 -) -> _ChatTemplateContentFormat:
441 - jinja_ast = _try_extract_ast(chat_template)
442 - if jinja_ast is None:
443 - return default
444 -
445 - try:
446 - next(_iter_nodes_assign_content_item(jinja_ast))
447 - except StopIteration:
448 - return "string"
449 - except Exception:
450 - logger.exception("Error when parsing AST of Jinja template")
451 - return default
452 - else:
453 - return "openai"
454 -
455 -
456 -def resolve_mistral_chat_template(
457 - chat_template: str | None,
458 - **kwargs: Any,
459 -) -> str | None:
460 - if chat_template is not None or kwargs.get("chat_template_kwargs") is not None:
461 - raise ValueError(
462 - "'chat_template' or 'chat_template_kwargs' cannot be overridden "
463 - "for mistral tokenizer."
464 - )
465 -
466 - return None
467 -
468 -
469 -_PROCESSOR_CHAT_TEMPLATES = dict[tuple[str, bool], str | None]()
470 -"""
471 -Used in `_try_get_processor_chat_template` to avoid calling
472 -`cached_get_processor` again if the processor fails to be loaded.
473 -
474 -This is needed because `lru_cache` does not cache when an exception happens.
475 -"""
476 -
477 -
478 -def _try_get_processor_chat_template(
479 - tokenizer: PreTrainedTokenizer | PreTrainedTokenizerFast,
480 - model_config: ModelConfig,
481 -) -> str | None:
482 - cache_key = (tokenizer.name_or_path, model_config.trust_remote_code)
483 - if cache_key in _PROCESSOR_CHAT_TEMPLATES:
484 - return _PROCESSOR_CHAT_TEMPLATES[cache_key]
485 -
486 - try:
487 - processor = cached_get_processor(
488 - tokenizer.name_or_path,
489 - processor_cls=(
490 - PreTrainedTokenizer,
491 - PreTrainedTokenizerFast,
492 - ProcessorMixin,
493 - ),
494 - trust_remote_code=model_config.trust_remote_code,
495 - )
496 - if (
497 - isinstance(processor, ProcessorMixin)
498 - and hasattr(processor, "chat_template")
499 - and (chat_template := processor.chat_template) is not None
500 - ):
501 - _PROCESSOR_CHAT_TEMPLATES[cache_key] = chat_template
502 - return chat_template
503 - except Exception:
504 - logger.debug(
505 - "Failed to load AutoProcessor chat template for %s",
506 - tokenizer.name_or_path,
507 - exc_info=True,
508 - )
509 -
510 - _PROCESSOR_CHAT_TEMPLATES[cache_key] = None
511 - return None
512 -
513 -
514 -def resolve_hf_chat_template(
515 - tokenizer: PreTrainedTokenizer | PreTrainedTokenizerFast,
516 - chat_template: str | None,
517 - tools: list[dict[str, Any]] | None,
518 - *,
519 - model_config: ModelConfig,
520 -) -> str | None:
521 - # 1st priority: The given chat template
522 - if chat_template is not None:
523 - return chat_template
524 -
525 - # 2nd priority: AutoProcessor chat template, unless tool calling is enabled
526 - if tools is None:
527 - chat_template = _try_get_processor_chat_template(tokenizer, model_config)
528 - if chat_template is not None:
529 - return chat_template
530 -
531 - # 3rd priority: AutoTokenizer chat template
532 - try:
533 - return tokenizer.get_chat_template(chat_template, tools=tools)
534 - except Exception:
535 - logger.debug(
536 - "Failed to load AutoTokenizer chat template for %s",
537 - tokenizer.name_or_path,
538 - exc_info=True,
539 - )
540 -
541 - # 4th priority: Predefined fallbacks
542 - path = get_chat_template_fallback_path(
543 - model_type=model_config.hf_config.model_type,
544 - tokenizer_name_or_path=model_config.tokenizer,
545 - )
546 - if path is not None:
547 - logger.info_once(
548 - "Loading chat template fallback for %s as there isn't one "
549 - "defined on HF Hub.",
550 - tokenizer.name_or_path,
551 - )
552 - chat_template = load_chat_template(path)
553 - else:
554 - logger.debug_once(
555 - "There is no chat template fallback for %s", tokenizer.name_or_path
556 - )
557 -
558 - return chat_template
559 -
560 -
561 -def _resolve_chat_template_content_format(
562 - chat_template: str | None,
563 - tools: list[dict[str, Any]] | None,
564 - tokenizer: TokenizerLike | None,
565 - *,
566 - model_config: ModelConfig,
567 -) -> _ChatTemplateContentFormat:
568 - if isinstance(tokenizer, (PreTrainedTokenizer, PreTrainedTokenizerFast)):
569 - hf_chat_template = resolve_hf_chat_template(
570 - tokenizer,
571 - chat_template=chat_template,
572 - tools=tools,
573 - model_config=model_config,
574 - )
575 - else:
576 - hf_chat_template = None
577 -
578 - jinja_text = (
579 - hf_chat_template
580 - if isinstance(hf_chat_template, str)
581 - else load_chat_template(chat_template, is_literal=True)
582 - )
583 -
584 - detected_format = (
585 - "string"
586 - if jinja_text is None
587 - else _detect_content_format(jinja_text, default="string")
588 - )
589 -
590 - return detected_format
591 -
592 -
593 -@lru_cache
594 -def _log_chat_template_content_format(
595 - chat_template: str | None,
596 - given_format: ChatTemplateContentFormatOption,
597 - detected_format: ChatTemplateContentFormatOption,
598 -):
599 - logger.info(
600 - "Detected the chat template content format to be '%s'. "
601 - "You can set `--chat-template-content-format` to override this.",
602 - detected_format,
603 - )
604 -
605 - if given_format != "auto" and given_format != detected_format:
606 - logger.warning(
607 - "You specified `--chat-template-content-format %s` "
608 - "which is different from the detected format '%s'. "
609 - "If our automatic detection is incorrect, please consider "
610 - "opening a GitHub issue so that we can improve it: "
611 - "https://github.com/vllm-project/vllm/issues/new/choose",
612 - given_format,
613 - detected_format,
614 - )
615 -
616 -
617 -def resolve_chat_template_content_format(
618 - chat_template: str | None,
619 - tools: list[dict[str, Any]] | None,
620 - given_format: ChatTemplateContentFormatOption,
621 - tokenizer: TokenizerLike | None,
622 - *,
623 - model_config: ModelConfig,
624 -) -> _ChatTemplateContentFormat:
625 - if given_format != "auto":
626 - return given_format
627 -
628 - detected_format = _resolve_chat_template_content_format(
629 - chat_template,
630 - tools,
631 - tokenizer,
632 - model_config=model_config,
633 - )
634 -
635 - _log_chat_template_content_format(
636 - chat_template,
637 - given_format=given_format,
638 - detected_format=detected_format,
639 - )
640 -
641 - return detected_format
326 +# After resolving "auto"
327 +ChatTemplateContentFormat = Literal["string", "openai"]
328
329
330 ModalityStr = Literal["image", "audio", "video", "image_embeds", "audio_embeds"]
@@ -1593,7 +1279,7 @@ _ToolParser = partial(cast, ChatCompletionToolMessageParam)
1279 def _parse_chat_message_content(
1280 message: ChatCompletionMessageParam,
1281 mm_tracker: BaseMultiModalItemTracker,
1596 - content_format: _ChatTemplateContentFormat,
1282 + content_format: ChatTemplateContentFormat,
1283 interleave_strings: bool,
1284 ) -> list[ConversationMessage]:
1285 role = message["role"]
@@ -1669,7 +1355,7 @@ def _postprocess_messages(messages: list[ConversationMessage]) -> None:
1355 def parse_chat_messages(
1356 messages: list[ChatCompletionMessageParam],
1357 model_config: ModelConfig,
1672 - content_format: _ChatTemplateContentFormat,
1358 + content_format: ChatTemplateContentFormat,
1359 ) -> tuple[
1360 list[ConversationMessage],
1361 MultiModalDataDict | None,
@@ -1697,13 +1383,13 @@ def parse_chat_messages(
1383 return conversation, mm_tracker.all_mm_data(), mm_tracker.all_mm_uuids()
1384
1385
1700 -def parse_chat_messages_futures(
1386 +async def parse_chat_messages_async(
1387 messages: list[ChatCompletionMessageParam],
1388 model_config: ModelConfig,
1703 - content_format: _ChatTemplateContentFormat,
1389 + content_format: ChatTemplateContentFormat,
1390 ) -> tuple[
1391 list[ConversationMessage],
1706 - Awaitable[MultiModalDataDict | None],
1392 + MultiModalDataDict | None,
1393 MultiModalUUIDDict | None,
1394 ]:
1395 conversation: list[ConversationMessage] = []
@@ -1725,174 +1411,7 @@ def parse_chat_messages_futures(
1411
1412 _postprocess_messages(conversation)
1413
1728 - return conversation, mm_tracker.all_mm_data(), mm_tracker.all_mm_uuids()
1729 -
1730 -
1731 -# adapted from https://github.com/huggingface/transformers/blob/v4.56.2/src/transformers/utils/chat_template_utils.py#L398-L412
1732 -# only preserve the parse function used to resolve chat template kwargs
1733 -class AssistantTracker(jinja2.ext.Extension):
1734 - tags = {"generation"}
1735 -
1736 - def parse(self, parser: jinja2.parser.Parser) -> jinja2.nodes.CallBlock:
1737 - lineno = next(parser.stream).lineno
1738 - body = parser.parse_statements(["name:endgeneration"], drop_needle=True)
1739 - call = self.call_method("_generation_support")
1740 - call_block = jinja2.nodes.CallBlock(call, [], [], body)
1741 - return call_block.set_lineno(lineno)
1742 -
1743 -
1744 -def _resolve_chat_template_kwargs(
1745 - chat_template: str,
1746 -):
1747 - env = jinja2.sandbox.ImmutableSandboxedEnvironment(
1748 - trim_blocks=True,
1749 - lstrip_blocks=True,
1750 - extensions=[AssistantTracker, jinja2.ext.loopcontrols],
1751 - )
1752 - parsed_content = env.parse(chat_template)
1753 - template_vars = jinja2.meta.find_undeclared_variables(parsed_content)
1754 - return template_vars
1755 -
1756 -
1757 -_cached_resolve_chat_template_kwargs = lru_cache(_resolve_chat_template_kwargs)
1758 -
1759 -
1760 -@lru_cache
1761 -def _get_hf_base_chat_template_params() -> frozenset[str]:
1762 - # Get standard parameters from HuggingFace's base tokenizer class.
1763 - # This dynamically extracts parameters from PreTrainedTokenizer's
1764 - # apply_chat_template method, ensuring compatibility with tokenizers
1765 - # that use **kwargs to receive standard parameters.
1766 -
1767 - # Read signature from HF's base class - the single source of truth
1768 - base_sig = inspect.signature(PreTrainedTokenizer.apply_chat_template)
1769 - # Exclude VAR_KEYWORD (**kwargs) and VAR_POSITIONAL (*args) placeholders
1770 - return frozenset(
1771 - p.name
1772 - for p in base_sig.parameters.values()
1773 - if p.kind
1774 - not in (inspect.Parameter.VAR_KEYWORD, inspect.Parameter.VAR_POSITIONAL)
1775 - )
1776 -
1777 -
1778 -def resolve_chat_template_kwargs(
1779 - tokenizer: PreTrainedTokenizer | PreTrainedTokenizerFast,
1780 - chat_template: str,
1781 - chat_template_kwargs: dict[str, Any],
1782 - raise_on_unexpected: bool = True,
1783 -) -> dict[str, Any]:
1784 - # We exclude chat_template from kwargs here, because
1785 - # chat template has been already resolved at this stage
1786 - unexpected_vars = {"chat_template", "tokenize"}
1787 - if raise_on_unexpected and (
1788 - unexpected_in_kwargs := unexpected_vars & chat_template_kwargs.keys()
1789 - ):
1790 - raise ValueError(
1791 - "Found unexpected chat template kwargs from request: "
1792 - f"{unexpected_in_kwargs}"
1793 - )
1794 -
1795 - fn_kw = {
1796 - k
1797 - for k in chat_template_kwargs
1798 - if supports_kw(tokenizer.apply_chat_template, k, allow_var_kwargs=False)
1799 - }
1800 - template_vars = _cached_resolve_chat_template_kwargs(chat_template)
1801 -
1802 - # Allow standard HF parameters even if tokenizer uses **kwargs to receive them
1803 - hf_base_params = _get_hf_base_chat_template_params()
1804 -
1805 - accept_vars = (fn_kw | template_vars | hf_base_params) - unexpected_vars
1806 - return {k: v for k, v in chat_template_kwargs.items() if k in accept_vars}
1807 -
1808 -
1809 -def apply_hf_chat_template(
1810 - tokenizer: PreTrainedTokenizer | PreTrainedTokenizerFast,
1811 - conversation: list[ConversationMessage],
1812 - chat_template: str | None,
1813 - tools: list[dict[str, Any]] | None,
1814 - *,
1815 - model_config: ModelConfig,
1816 - **kwargs: Any,
1817 -) -> str:
1818 - hf_chat_template = resolve_hf_chat_template(
1819 - tokenizer,
1820 - chat_template=chat_template,
1821 - tools=tools,
1822 - model_config=model_config,
1823 - )
1824 -
1825 - if hf_chat_template is None:
1826 - raise ChatTemplateResolutionError(
1827 - "As of transformers v4.44, default chat template is no longer "
1828 - "allowed, so you must provide a chat template if the tokenizer "
1829 - "does not define one."
1830 - )
1831 -
1832 - resolved_kwargs = resolve_chat_template_kwargs(
1833 - tokenizer=tokenizer,
1834 - chat_template=hf_chat_template,
1835 - chat_template_kwargs=kwargs,
1836 - )
1837 -
1838 - try:
1839 - return tokenizer.apply_chat_template(
1840 - conversation=conversation, # type: ignore[arg-type]
1841 - tools=tools, # type: ignore[arg-type]
1842 - chat_template=hf_chat_template,
1843 - tokenize=False,
1844 - **resolved_kwargs,
1845 - )
1846 -
1847 - # External library exceptions can sometimes occur despite the framework's
1848 - # internal exception management capabilities.
1849 - except Exception as e:
1850 - # Log and report any library-related exceptions for further
1851 - # investigation.
1852 - logger.exception(
1853 - "An error occurred in `transformers` while applying chat template"
1854 - )
1855 - raise ValueError(str(e)) from e
1856 -
1857 -
1858 -def apply_mistral_chat_template(
1859 - tokenizer: "MistralTokenizer",
1860 - messages: list[ChatCompletionMessageParam],
1861 - chat_template: str | None,
1862 - tools: list[dict[str, Any]] | None,
1863 - **kwargs: Any,
1864 -) -> list[int]:
1865 - from mistral_common.exceptions import MistralCommonException
1866 -
1867 - # The return value of resolve_mistral_chat_template is always None,
1868 - # and we won't use it.
1869 - resolve_mistral_chat_template(
1870 - chat_template=chat_template,
1871 - **kwargs,
1872 - )
1873 -
1874 - try:
1875 - return tokenizer.apply_chat_template(
1876 - messages=messages,
1877 - tools=tools,
1878 - **kwargs,
1879 - )
1880 - # mistral-common uses assert statements to stop processing of input
1881 - # if input does not comply with the expected format.
1882 - # We convert those assertion errors to ValueErrors so they can be
1883 - # properly caught in the preprocessing_input step
1884 - except (AssertionError, MistralCommonException) as e:
1885 - raise ValueError(str(e)) from e
1886 -
1887 - # External library exceptions can sometimes occur despite the framework's
1888 - # internal exception management capabilities.
1889 - except Exception as e:
1890 - # Log and report any library-related exceptions for further
1891 - # investigation.
1892 - logger.exception(
1893 - "An error occurred in `mistral_common` while applying chat template"
1894 - )
1895 - raise ValueError(str(e)) from e
1414 + return conversation, await mm_tracker.all_mm_data(), mm_tracker.all_mm_uuids()
1415
1416
1417 def get_history_tool_calls_cnt(conversation: list[ConversationMessage]):
vllm/entrypoints/llm.py
+14 -54
@@ -37,10 +37,6 @@ from vllm.engine.arg_utils import EngineArgs
37 from vllm.entrypoints.chat_utils import (
38 ChatCompletionMessageParam,
39 ChatTemplateContentFormatOption,
40 - apply_hf_chat_template,
41 - apply_mistral_chat_template,
42 - parse_chat_messages,
43 - resolve_chat_template_content_format,
40 )
41 from vllm.entrypoints.pooling.score.utils import (
42 ScoreContentPartParam,
@@ -786,7 +782,7 @@ class LLM:
782 tools: list[dict[str, Any]] | None = None,
783 chat_template_kwargs: dict[str, Any] | None = None,
784 mm_processor_kwargs: dict[str, Any] | None = None,
789 - ) -> list[TokensPrompt]:
785 + ) -> list[TextPrompt | TokensPrompt]:
786 """
787 Generate prompt for a chat conversation. The pre-processed
788 prompt can then be used as input for the other LLM methods.
@@ -807,63 +803,27 @@ class LLM:
803 # messages is list[...]
804 list_of_messages = [cast(list[ChatCompletionMessageParam], messages)]
805
810 - tokenizer = self.get_tokenizer()
811 - model_config = self.model_config
812 - resolved_content_format = resolve_chat_template_content_format(
813 - chat_template,
814 - tools,
815 - chat_template_content_format,
816 - tokenizer,
817 - model_config=model_config,
818 - )
806 + renderer = self.llm_engine.renderer
807
820 - _chat_template_kwargs: dict[str, Any] = dict(
821 - chat_template=chat_template,
822 - add_generation_prompt=add_generation_prompt,
823 - continue_final_message=continue_final_message,
824 - tools=tools,
825 - )
826 - _chat_template_kwargs.update(chat_template_kwargs or {})
808 + chat_template_kwargs = {
809 + "chat_template": chat_template,
810 + "add_generation_prompt": add_generation_prompt,
811 + "continue_final_message": continue_final_message,
812 + "tools": tools,
813 + **(chat_template_kwargs or {}),
814 + }
815
828 - prompts: list[TokensPrompt] = []
816 + prompts = list[TextPrompt | TokensPrompt]()
817
818 for msgs in list_of_messages:
831 - # NOTE: _parse_chat_message_content_parts() currently doesn't
819 + # NOTE: renderer.render_messages() currently doesn't
820 # handle mm_processor_kwargs, since there is no implementation in
821 # the chat message parsing for it.
834 - conversation, mm_data, mm_uuids = parse_chat_messages(
822 + _, prompt = renderer.render_messages(
823 msgs,
836 - model_config,
837 - content_format=resolved_content_format,
824 + chat_template_content_format=chat_template_content_format,
825 + **chat_template_kwargs,
826 )
839 -
840 - if isinstance(tokenizer, MistralTokenizer):
841 - prompt_token_ids = apply_mistral_chat_template(
842 - tokenizer,
843 - messages=msgs,
844 - **_chat_template_kwargs,
845 - )
846 - else:
847 - prompt_str = apply_hf_chat_template(
848 - tokenizer=tokenizer,
849 - conversation=conversation,
850 - model_config=model_config,
851 - **_chat_template_kwargs,
852 - )
853 - # Special tokens are already included in chat templates so
854 - # should not be added by the tokenizer in this case.
855 - prompt_token_ids = tokenizer.encode(
856 - prompt_str, add_special_tokens=False
857 - )
858 -
859 - prompt = TokensPrompt(prompt_token_ids=prompt_token_ids)
860 -
861 - if mm_data is not None:
862 - prompt["multi_modal_data"] = mm_data
863 -
864 - if mm_uuids is not None:
865 - prompt["multi_modal_uuids"] = mm_uuids
866 -
827 if mm_processor_kwargs is not None:
828 prompt["mm_processor_kwargs"] = mm_processor_kwargs
829
vllm/entrypoints/openai/api_server.py
+2 -4
@@ -34,6 +34,7 @@ import vllm.envs as envs
34 from vllm.engine.arg_utils import AsyncEngineArgs
35 from vllm.engine.protocol import EngineClient
36 from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
37 +from vllm.entrypoints.chat_utils import load_chat_template
38 from vllm.entrypoints.launcher import serve_http
39 from vllm.entrypoints.logger import RequestLogger
40 from vllm.entrypoints.mcp.tool_server import DemoToolServer, MCPToolServer, ToolServer
@@ -62,7 +63,6 @@ from vllm.entrypoints.serve.tokenize.serving import OpenAIServingTokenization
63 from vllm.entrypoints.utils import (
64 cli_env_setup,
65 log_non_default_args,
65 - process_chat_template,
66 process_lora_modules,
67 sanitize_message,
68 )
@@ -662,9 +662,7 @@ async def init_app_state(
662 supported_tasks = await engine_client.get_supported_tasks()
663 logger.info("Supported tasks: %s", supported_tasks)
664
665 - resolved_chat_template = await process_chat_template(
666 - args.chat_template, engine_client, vllm_config.model_config
667 - )
665 + resolved_chat_template = load_chat_template(args.chat_template)
666
667 if args.tool_server == "demo":
668 tool_server: ToolServer | None = DemoToolServer()
vllm/entrypoints/openai/chat_completion/serving.py
+8 -8
@@ -186,8 +186,7 @@ class OpenAIServingChat(OpenAIServing):
186 start_time = time.perf_counter()
187
188 try:
189 - # Get the tokenizer from the engine
190 - tokenizer = await self.engine_client.get_tokenizer()
189 + renderer = self.engine_client.renderer
190
191 # Create a minimal dummy request
192 dummy_request = ChatCompletionRequest(
@@ -203,7 +202,7 @@ class OpenAIServingChat(OpenAIServing):
202 # 3. Tokenizer initialization for chat
203 await self._preprocess_chat(
204 dummy_request,
206 - tokenizer,
205 + renderer,
206 dummy_request.messages,
207 chat_template=self.chat_template,
208 chat_template_content_format=self.chat_template_content_format,
@@ -247,7 +246,8 @@ class OpenAIServingChat(OpenAIServing):
246 raise self.engine_client.dead_error
247
248 try:
250 - tokenizer = await self.engine_client.get_tokenizer()
249 + renderer = self.engine_client.renderer
250 + tokenizer = renderer.tokenizer
251
252 tool_parser = self.tool_parser
253
@@ -308,7 +308,7 @@ class OpenAIServingChat(OpenAIServing):
308
309 conversation, engine_prompts = await self._preprocess_chat(
310 request,
311 - tokenizer,
311 + renderer,
312 request.messages,
313 chat_template=request.chat_template or self.chat_template,
314 chat_template_content_format=self.chat_template_content_format,
@@ -365,8 +365,6 @@ class OpenAIServingChat(OpenAIServing):
365 )
366
367 model_name = self.models.model_name(lora_request)
368 -
369 - tokenizer = await self.engine_client.get_tokenizer()
368 except (ValueError, TypeError, RuntimeError) as e:
369 logger.exception("Error preparing request components")
370 return self.create_error_response(e)
@@ -463,6 +461,8 @@ class OpenAIServingChat(OpenAIServing):
461 (result_generator,) = generators
462
463 # Streaming response
464 + tokenizer = self.renderer.tokenizer
465 +
466 if request.stream:
467 return self.chat_completion_stream_generator(
468 request,
@@ -1784,7 +1784,7 @@ class OpenAIServingChat(OpenAIServing):
1784 else:
1785 if tokenizer is None:
1786 raise ValueError(
1787 - "Tokenizer not available when `skip_tokenizer_init=True`"
1787 + "Unable to get tokenizer because `skip_tokenizer_init=True`"
1788 )
1789
1790 token = tokenizer.decode(token_id)
vllm/entrypoints/openai/completion/serving.py
+3 -11
@@ -117,12 +117,7 @@ class OpenAIServingCompletion(OpenAIServing):
117 )
118
119 try:
120 - if self.model_config.skip_tokenizer_init:
121 - tokenizer = None
122 - else:
123 - tokenizer = await self.engine_client.get_tokenizer()
124 - renderer = self._get_renderer(tokenizer)
125 -
120 + renderer = self._get_completion_renderer()
121 engine_prompts = await renderer.render_prompt_and_embeds(
122 prompt_or_prompts=request.prompt,
123 prompt_embeds=request.prompt_embeds,
@@ -163,11 +158,6 @@ class OpenAIServingCompletion(OpenAIServing):
158
159 try:
160 lora_request = self._maybe_get_adapters(request)
166 -
167 - if self.model_config.skip_tokenizer_init:
168 - tokenizer = None
169 - else:
170 - tokenizer = await self.engine_client.get_tokenizer()
161 except (ValueError, TypeError, RuntimeError) as e:
162 logger.exception("Error preparing request components")
163 return self.create_error_response(e)
@@ -280,6 +270,8 @@ class OpenAIServingCompletion(OpenAIServing):
270 stream = request.stream and not request.use_beam_search
271
272 # Streaming response
273 + tokenizer = self.renderer.tokenizer
274 +
275 if stream:
276 return self.completion_stream_generator(
277 request,
vllm/entrypoints/openai/engine/serving.py
+54 -109
@@ -6,10 +6,9 @@ import sys
6 import time
7 import traceback
8 from collections.abc import AsyncGenerator, Callable, Iterable, Mapping
9 -from concurrent.futures import ThreadPoolExecutor
9 from dataclasses import dataclass, field
10 from http import HTTPStatus
12 -from typing import Any, ClassVar, Generic, TypeAlias, TypeVar
11 +from typing import Any, ClassVar, Generic, TypeAlias, TypeVar, cast
12
13 import numpy as np
14 from fastapi import Request
@@ -26,10 +25,6 @@ from vllm.entrypoints.chat_utils import (
25 ChatCompletionMessageParam,
26 ChatTemplateContentFormatOption,
27 ConversationMessage,
29 - apply_hf_chat_template,
30 - apply_mistral_chat_template,
31 - parse_chat_messages_futures,
32 - resolve_chat_template_content_format,
28 )
29 from vllm.entrypoints.logger import RequestLogger
30 from vllm.entrypoints.openai.chat_completion.protocol import (
@@ -113,10 +108,9 @@ from vllm.multimodal import MultiModalDataDict
108 from vllm.outputs import CompletionOutput, PoolingRequestOutput, RequestOutput
109 from vllm.pooling_params import PoolingParams
110 from vllm.reasoning import ReasoningParser, ReasoningParserManager
111 +from vllm.renderers import RendererLike
112 from vllm.sampling_params import BeamSearchParams, SamplingParams
113 from vllm.tokenizers import TokenizerLike
118 -from vllm.tokenizers.deepseek_v32 import DeepseekV32Tokenizer
119 -from vllm.tokenizers.mistral import MistralTokenizer
114 from vllm.tool_parsers import ToolParser, ToolParserManager
115 from vllm.tracing import (
116 contains_trace_headers,
@@ -127,10 +121,8 @@ from vllm.utils import random_uuid
121 from vllm.utils.async_utils import (
122 AsyncMicrobatchTokenizer,
123 collect_from_async_generator,
130 - make_async,
124 merge_async_iterators,
125 )
133 -from vllm.utils.collection_utils import is_list_of
126 from vllm.v1.engine import EngineCoreRequest
127
128
@@ -215,7 +207,6 @@ class ResponseGenerationMixin:
207
208 @dataclass(kw_only=True)
209 class ServeContext(RequestProcessingMixin, ResponseGenerationMixin, Generic[RequestT]):
218 - # Shared across all requests
210 request: RequestT
211 raw_request: Request | None = None
212 model_name: str
@@ -223,9 +214,6 @@ class ServeContext(RequestProcessingMixin, ResponseGenerationMixin, Generic[Requ
214 created_time: int = field(default_factory=lambda: int(time.time()))
215 lora_request: LoRARequest | None = None
216
226 - # Shared across most requests
227 - tokenizer: TokenizerLike | None = None
228 -
217
218 @dataclass(kw_only=True)
219 class ClassificationServeContext(ServeContext[ClassificationRequest]):
@@ -261,16 +249,13 @@ class OpenAIServing:
249
250 self.request_logger = request_logger
251 self.return_tokens_as_token_ids = return_tokens_as_token_ids
264 - self._tokenizer_executor = ThreadPoolExecutor(max_workers=1)
265 - self._apply_mistral_chat_template_async = make_async(
266 - apply_mistral_chat_template, executor=self._tokenizer_executor
267 - )
252
253 self._async_tokenizer_pool: dict[TokenizerLike, AsyncMicrobatchTokenizer] = {}
254 self.log_error_stack = log_error_stack
255
256 self.input_processor = self.models.input_processor
257 self.io_processor = self.models.io_processor
258 + self.renderer = self.models.renderer
259 self.model_config = self.models.model_config
260 self.max_model_len = self.model_config.max_model_len
261
@@ -557,14 +542,14 @@ class OpenAIServing:
542 prompt_logprobs=None,
543 )
544
560 - def _get_renderer(self, tokenizer: TokenizerLike | None) -> BaseRenderer:
545 + def _get_completion_renderer(self) -> BaseRenderer:
546 """
547 Get a Renderer instance with the provided tokenizer.
548 Uses shared async tokenizer pool for efficiency.
549 """
550 return CompletionRenderer(
551 model_config=self.model_config,
567 - tokenizer=tokenizer,
552 + tokenizer=self.renderer.tokenizer,
553 async_tokenizer_pool=self._async_tokenizer_pool,
554 )
555
@@ -1183,7 +1168,7 @@ class OpenAIServing:
1168 async def _preprocess_chat(
1169 self,
1170 request: ChatLikeRequest | ResponsesRequest,
1186 - tokenizer: TokenizerLike | None,
1171 + renderer: RendererLike,
1172 messages: list[ChatCompletionMessageParam],
1173 chat_template: str | None,
1174 chat_template_content_format: ChatTemplateContentFormatOption,
@@ -1196,59 +1181,58 @@ class OpenAIServing:
1181 tool_parser: Callable[[TokenizerLike], ToolParser] | None = None,
1182 add_special_tokens: bool = False,
1183 ) -> tuple[list[ConversationMessage], list[TokensPrompt]]:
1199 - model_config = self.model_config
1200 -
1201 - resolved_content_format = resolve_chat_template_content_format(
1202 - chat_template,
1203 - tool_dicts,
1204 - chat_template_content_format,
1205 - tokenizer,
1206 - model_config=model_config,
1207 - )
1208 - conversation, mm_data_future, mm_uuids = parse_chat_messages_futures(
1209 - messages,
1210 - model_config,
1211 - content_format=resolved_content_format,
1212 - )
1213 -
1214 - _chat_template_kwargs: dict[str, Any] = dict(
1215 - chat_template=chat_template,
1216 - add_generation_prompt=add_generation_prompt,
1217 - continue_final_message=continue_final_message,
1218 - tools=tool_dicts,
1219 - documents=documents,
1220 - )
1221 - _chat_template_kwargs |= self._prepare_extra_chat_template_kwargs(
1184 + chat_template_kwargs = {
1185 + "chat_template": chat_template,
1186 + "add_generation_prompt": add_generation_prompt,
1187 + "continue_final_message": continue_final_message,
1188 + "tools": tool_dicts,
1189 + "documents": documents,
1190 + **(chat_template_kwargs or {}),
1191 + }
1192 + chat_template_kwargs = self._prepare_extra_chat_template_kwargs(
1193 chat_template_kwargs,
1194 default_chat_template_kwargs,
1195 )
1196
1226 - request_prompt: str | list[int]
1197 + # Use the async tokenizer in `OpenAIServing` if possible.
1198 + # Later we can move it into the renderer so that we can return both
1199 + # text and token IDs in the same prompt from `render_messages_async`
1200 + # which is used for logging and `enable_response_messages`.
1201 + from vllm.tokenizers.mistral import MistralTokenizer
1202
1228 - if tokenizer is None:
1229 - request_prompt = "placeholder"
1230 - elif isinstance(tokenizer, MistralTokenizer):
1231 - request_prompt = await self._apply_mistral_chat_template_async(
1232 - tokenizer,
1233 - messages=messages,
1234 - **_chat_template_kwargs,
1235 - )
1236 - elif isinstance(tokenizer, DeepseekV32Tokenizer):
1237 - request_prompt = tokenizer.apply_chat_template(
1238 - conversation=conversation,
1239 - messages=messages,
1240 - model_config=model_config,
1241 - **_chat_template_kwargs,
1203 + conversation, engine_prompt = await renderer.render_messages_async(
1204 + messages,
1205 + chat_template_content_format=chat_template_content_format,
1206 + tokenize=(
1207 + chat_template_kwargs.pop("tokenize", False)
1208 + or isinstance(renderer.tokenizer, MistralTokenizer)
1209 + ),
1210 + **chat_template_kwargs,
1211 + )
1212 +
1213 + if "prompt_token_ids" not in engine_prompt:
1214 + extra_data = engine_prompt
1215 + engine_prompt = await self._tokenize_prompt_input_async(
1216 + request,
1217 + renderer.get_tokenizer(),
1218 + engine_prompt["prompt"],
1219 + add_special_tokens=add_special_tokens,
1220 )
1221 + # Fill in other keys like MM data
1222 + engine_prompt.update(extra_data) # type: ignore
1223 else:
1244 - request_prompt = apply_hf_chat_template(
1245 - tokenizer=tokenizer,
1246 - conversation=conversation,
1247 - model_config=model_config,
1248 - **_chat_template_kwargs,
1224 + self._validate_input(
1225 + request=request,
1226 + input_ids=engine_prompt["prompt_token_ids"], # type: ignore
1227 + input_text="",
1228 )
1229
1251 - mm_data = await mm_data_future
1230 + engine_prompt = cast(TokensPrompt, engine_prompt)
1231 +
1232 + if request.mm_processor_kwargs is not None:
1233 + engine_prompt["mm_processor_kwargs"] = request.mm_processor_kwargs
1234 + if (cache_salt := getattr(request, "cache_salt", None)) is not None:
1235 + engine_prompt["cache_salt"] = cache_salt
1236
1237 # tool parsing is done only if a tool_parser has been set and if
1238 # tool_choice is not "none" (if tool_choice is "none" but a tool_parser
@@ -1264,48 +1248,9 @@ class OpenAIServing:
1248 "or Responses API requests."
1249 )
1250 raise NotImplementedError(msg)
1267 - request = tool_parser(tokenizer).adjust_request(request=request) # type: ignore
1268 -
1269 - if tokenizer is None:
1270 - assert isinstance(request_prompt, str), (
1271 - "Prompt has to be a string",
1272 - "when the tokenizer is not initialised",
1273 - )
1274 - prompt_inputs = TokensPrompt(prompt=request_prompt, prompt_token_ids=[1])
1275 - elif isinstance(request_prompt, str):
1276 - prompt_inputs = await self._tokenize_prompt_input_async(
1277 - request,
1278 - tokenizer,
1279 - request_prompt,
1280 - add_special_tokens=add_special_tokens,
1281 - )
1282 - else:
1283 - # For MistralTokenizer
1284 - assert is_list_of(request_prompt, int), (
1285 - "Prompt has to be either a string or a list of token ids"
1286 - )
1287 - input_text = tokenizer.decode(request_prompt)
1288 - prompt_inputs = self._validate_input(
1289 - request=request,
1290 - input_ids=request_prompt,
1291 - input_text=input_text,
1292 - )
1293 -
1294 - engine_prompt = TokensPrompt(prompt_token_ids=prompt_inputs["prompt_token_ids"])
1295 - if "prompt" in prompt_inputs:
1296 - engine_prompt["prompt"] = prompt_inputs["prompt"]
1251
1298 - if mm_data is not None:
1299 - engine_prompt["multi_modal_data"] = mm_data
1300 -
1301 - if mm_uuids is not None:
1302 - engine_prompt["multi_modal_uuids"] = mm_uuids
1303 -
1304 - if request.mm_processor_kwargs is not None:
1305 - engine_prompt["mm_processor_kwargs"] = request.mm_processor_kwargs
1306 -
1307 - if hasattr(request, "cache_salt") and request.cache_salt is not None:
1308 - engine_prompt["cache_salt"] = request.cache_salt
1252 + tokenizer = renderer.get_tokenizer()
1253 + request = tool_parser(tokenizer).adjust_request(request=request) # type: ignore
1254
1255 return conversation, [engine_prompt]
1256
@@ -1341,7 +1286,7 @@ class OpenAIServing:
1286 async def _render_next_turn(
1287 self,
1288 request: ResponsesRequest,
1344 - tokenizer: TokenizerLike | None,
1289 + renderer: RendererLike,
1290 messages: list[ResponseInputOutputItem],
1291 tool_dicts: list[dict[str, Any]] | None,
1292 tool_parser,
@@ -1354,7 +1299,7 @@ class OpenAIServing:
1299
1300 _, engine_prompts = await self._preprocess_chat(
1301 request,
1357 - tokenizer,
1302 + renderer,
1303 new_messages,
1304 tool_dicts=tool_dicts,
1305 tool_parser=tool_parser,
@@ -1431,7 +1376,7 @@ class OpenAIServing:
1376 elif isinstance(context, ParsableContext):
1377 engine_prompts = await self._render_next_turn(
1378 context.request,
1434 - context.tokenizer,
1379 + context.renderer,
1380 context.parser.response_messages,
1381 context.tool_dicts,
1382 context.tool_parser_cls,
vllm/entrypoints/openai/models/serving.py
+1
@@ -61,6 +61,7 @@ class OpenAIServingModels:
61
62 self.input_processor = self.engine_client.input_processor
63 self.io_processor = self.engine_client.io_processor
64 + self.renderer = self.engine_client.renderer
65 self.model_config = self.engine_client.model_config
66 self.max_model_len = self.model_config.max_model_len
67
vllm/entrypoints/openai/responses/context.py
+4 -1
@@ -43,6 +43,7 @@ from vllm.entrypoints.openai.responses.protocol import (
43 from vllm.entrypoints.openai.responses.utils import construct_tool_dicts
44 from vllm.outputs import RequestOutput
45 from vllm.reasoning.abs_reasoning_parsers import ReasoningParser
46 +from vllm.renderers import RendererLike
47 from vllm.tokenizers import TokenizerLike
48 from vllm.tool_parsers.abstract_tool_parser import ToolParser
49 from vllm.utils import random_uuid
@@ -260,7 +261,7 @@ class ParsableContext(ConversationContext):
261 self,
262 *,
263 response_messages: list[ResponseInputOutputItem],
263 - tokenizer: TokenizerLike,
264 + renderer: RendererLike,
265 reasoning_parser_cls: Callable[[TokenizerLike], ReasoningParser] | None,
266 request: ResponsesRequest,
267 available_tools: list[str] | None,
@@ -279,6 +280,7 @@ class ParsableContext(ConversationContext):
280 if reasoning_parser_cls is None:
281 raise ValueError("reasoning_parser_cls must be provided.")
282
283 + tokenizer = renderer.get_tokenizer()
284 self.parser = get_responses_parser_for_simple_context(
285 tokenizer=tokenizer,
286 reasoning_parser_cls=reasoning_parser_cls,
@@ -288,6 +290,7 @@ class ParsableContext(ConversationContext):
290 )
291 self.tool_parser_cls = tool_parser_cls
292 self.request = request
293 + self.renderer = renderer
294 self.tokenizer = tokenizer
295
296 self.available_tools = available_tools or []
vllm/entrypoints/openai/responses/serving.py
+8 -5
@@ -121,6 +121,7 @@ from vllm.logger import init_logger
121 from vllm.logprobs import Logprob as SampleLogprob
122 from vllm.logprobs import SampleLogprobs
123 from vllm.outputs import CompletionOutput
124 +from vllm.renderers import RendererLike
125 from vllm.sampling_params import SamplingParams, StructuredOutputsParams
126 from vllm.tokenizers import TokenizerLike
127 from vllm.utils import random_uuid
@@ -380,7 +381,8 @@ class OpenAIServingResponses(OpenAIServing):
381 try:
382 lora_request = self._maybe_get_adapters(request)
383 model_name = self.models.model_name(lora_request)
383 - tokenizer = await self.engine_client.get_tokenizer()
384 + renderer = self.engine_client.renderer
385 + tokenizer = renderer.get_tokenizer()
386
387 if self.use_harmony:
388 messages, engine_prompts = self._make_request_with_harmony(
@@ -388,7 +390,7 @@ class OpenAIServingResponses(OpenAIServing):
390 )
391 else:
392 messages, engine_prompts = await self._make_request(
391 - request, prev_response, tokenizer
393 + request, prev_response, renderer
394 )
395
396 except (
@@ -454,7 +456,7 @@ class OpenAIServingResponses(OpenAIServing):
456 # tokens during generation instead of at the end
457 context = ParsableContext(
458 response_messages=messages,
457 - tokenizer=tokenizer,
459 + renderer=renderer,
460 reasoning_parser_cls=self.reasoning_parser,
461 request=request,
462 tool_parser_cls=self.tool_parser,
@@ -585,7 +587,7 @@ class OpenAIServingResponses(OpenAIServing):
587 self,
588 request: ResponsesRequest,
589 prev_response: ResponsesResponse | None,
588 - tokenizer: TokenizerLike,
590 + renderer: RendererLike,
591 ):
592 tool_dicts = construct_tool_dicts(request.tools, request.tool_choice)
593 # Construct the input messages.
@@ -607,7 +609,7 @@ class OpenAIServingResponses(OpenAIServing):
609
610 _, engine_prompts = await self._preprocess_chat(
611 request,
610 - tokenizer,
612 + renderer,
613 messages,
614 tool_dicts=tool_dicts,
615 tool_parser=self.tool_parser,
@@ -631,6 +633,7 @@ class OpenAIServingResponses(OpenAIServing):
633 raise NotImplementedError(
634 "Only 'auto' tool_choice is supported in response API with Harmony"
635 )
636 +
637 messages = self._construct_input_messages_with_harmony(request, prev_response)
638 prompt_token_ids = render_for_completion(messages)
639 engine_prompt = TokensPrompt(prompt_token_ids=prompt_token_ids)
vllm/entrypoints/pooling/__init__.py
+2 -6
@@ -28,21 +28,17 @@ def register_pooling_api_routers(app: FastAPI):
28 async def init_pooling_state(
29 engine_client: "EngineClient", state: "State", args: "Namespace"
30 ):
31 + from vllm.entrypoints.chat_utils import load_chat_template
32 from vllm.entrypoints.logger import RequestLogger
33 from vllm.entrypoints.pooling.classify.serving import ServingClassification
34 from vllm.entrypoints.pooling.embed.serving import OpenAIServingEmbedding
35 from vllm.entrypoints.pooling.pooling.serving import OpenAIServingPooling
36 from vllm.entrypoints.pooling.score.serving import ServingScores
36 - from vllm.entrypoints.utils import process_chat_template
37 from vllm.tasks import POOLING_TASKS
38
39 supported_tasks = await engine_client.get_supported_tasks()
40
41 - vllm_config = engine_client.vllm_config
42 -
43 - resolved_chat_template = await process_chat_template(
44 - args.chat_template, engine_client, vllm_config.model_config
45 - )
41 + resolved_chat_template = load_chat_template(args.chat_template)
42
43 if args.enable_log_requests:
44 request_logger = RequestLogger(max_log_len=args.max_log_len)
vllm/entrypoints/pooling/classify/serving.py
+2 -4
@@ -54,8 +54,6 @@ class ClassificationMixin(OpenAIServing):
54 """
55 ctx = cast(ClassificationServeContext, ctx)
56 try:
57 - ctx.tokenizer = await self.engine_client.get_tokenizer()
58 -
57 request_obj = ctx.request
58
59 if isinstance(request_obj, ClassificationChatRequest):
@@ -76,7 +74,7 @@ class ClassificationMixin(OpenAIServing):
74
75 _, engine_prompts = await self._preprocess_chat(
76 cast(ChatCompletionRequest, chat_request),
79 - ctx.tokenizer,
77 + self.renderer,
78 messages,
79 chat_template=(
80 chat_request.chat_template
@@ -104,7 +102,7 @@ class ClassificationMixin(OpenAIServing):
102 ctx.engine_prompts = []
103 return None
104
107 - renderer = self._get_renderer(ctx.tokenizer)
105 + renderer = self._get_completion_renderer()
106 prompt_input = cast(str | list[str], input_data)
107 ctx.engine_prompts = await renderer.render_prompt(
108 prompt_or_prompts=prompt_input,
vllm/entrypoints/pooling/embed/serving.py
+2 -4
@@ -78,13 +78,10 @@ class EmbeddingMixin(OpenAIServing):
78 try:
79 ctx.lora_request = self._maybe_get_adapters(ctx.request)
80
81 - tokenizer = await self.engine_client.get_tokenizer()
82 - renderer = self._get_renderer(tokenizer)
83 -
81 if isinstance(ctx.request, EmbeddingChatRequest):
82 _, ctx.engine_prompts = await self._preprocess_chat(
83 ctx.request,
87 - tokenizer,
84 + self.renderer,
85 ctx.request.messages,
86 chat_template=ctx.request.chat_template or ctx.chat_template,
87 chat_template_content_format=ctx.chat_template_content_format,
@@ -93,6 +90,7 @@ class EmbeddingMixin(OpenAIServing):
90 add_special_tokens=ctx.request.add_special_tokens,
91 )
92 else:
93 + renderer = self._get_completion_renderer()
94 ctx.engine_prompts = await renderer.render_prompt(
95 prompt_or_prompts=ctx.request.input,
96 config=self._build_render_config(ctx.request),
vllm/entrypoints/pooling/pooling/serving.py
+2 -7
@@ -94,12 +94,6 @@ class OpenAIServingPooling(OpenAIServing):
94 try:
95 lora_request = self._maybe_get_adapters(request)
96
97 - if self.model_config.skip_tokenizer_init:
98 - tokenizer = None
99 - else:
100 - tokenizer = await self.engine_client.get_tokenizer()
101 - renderer = self._get_renderer(tokenizer)
102 -
97 if getattr(request, "dimensions", None) is not None:
98 return self.create_error_response(
99 "dimensions is currently not supported"
@@ -140,7 +134,7 @@ class OpenAIServingPooling(OpenAIServing):
134
135 _, engine_prompts = await self._preprocess_chat(
136 request,
143 - tokenizer,
137 + self.renderer,
138 request.messages,
139 chat_template=request.chat_template or self.chat_template,
140 chat_template_content_format=self.chat_template_content_format,
@@ -149,6 +143,7 @@ class OpenAIServingPooling(OpenAIServing):
143 add_special_tokens=request.add_special_tokens,
144 )
145 elif isinstance(request, PoolingCompletionRequest):
146 + renderer = self._get_completion_renderer()
147 engine_prompts = await renderer.render_prompt(
148 prompt_or_prompts=request.input,
149 config=self._build_render_config(request),
vllm/entrypoints/pooling/score/serving.py
+4 -2
@@ -3,6 +3,7 @@
3 import asyncio
4 import time
5 from collections.abc import AsyncGenerator, Mapping
6 +from concurrent.futures import ThreadPoolExecutor
7 from typing import Any
8
9 from fastapi import Request
@@ -63,6 +64,8 @@ class ServingScores(OpenAIServing):
64 )
65 self.score_template = score_template
66
67 + self._tokenizer_executor = ThreadPoolExecutor(max_workers=1)
68 +
69 async def _embedding_score(
70 self,
71 tokenizer: TokenizerLike,
@@ -283,8 +286,7 @@ class ServingScores(OpenAIServing):
286 raw_request: Request | None = None,
287 ) -> list[PoolingRequestOutput] | ErrorResponse:
288 lora_request = self._maybe_get_adapters(request)
286 -
287 - tokenizer = await self.engine_client.get_tokenizer()
289 + tokenizer = self.renderer.get_tokenizer()
290
291 truncate_prompt_tokens = getattr(request, "truncate_prompt_tokens", None)
292
vllm/entrypoints/pooling/score/utils.py
+5 -4
@@ -16,12 +16,12 @@ from vllm.entrypoints.chat_utils import (
16 MultiModalItemTracker,
17 _ContentPart,
18 _parse_chat_message_content_part,
19 - apply_hf_chat_template,
19 )
20 from vllm.inputs import TokensPrompt
21 from vllm.model_executor.models.interfaces import supports_score_template
22 from vllm.multimodal.inputs import MultiModalDataDict
23 from vllm.outputs import PoolingRequestOutput
24 +from vllm.renderers.hf import safe_apply_chat_template
25 from vllm.tokenizers import TokenizerLike
26
27 ScoreContentPartParam: TypeAlias = (
@@ -224,15 +224,16 @@ def get_score_prompt(
224 # If that fails because there is no such template,
225 # fall back to the default implementation.
226 try:
227 - full_prompt = apply_hf_chat_template(
227 + full_prompt = safe_apply_chat_template(
228 + model_config,
229 tokenizer,
230 [
231 {"role": "query", "content": prompt_1},
232 {"role": "document", "content": prompt_2},
233 ],
233 - score_template,
234 + chat_template=score_template,
235 tools=None,
235 - model_config=model_config,
236 + tokenize=False,
237 )
238 prompt_inputs = tokenizer(full_prompt, **tokenization_kwargs)
239 except ChatTemplateResolutionError:
vllm/entrypoints/serve/tokenize/serving.py
+5 -7
@@ -67,9 +67,6 @@ class OpenAIServingTokenization(OpenAIServing):
67 try:
68 lora_request = self._maybe_get_adapters(request)
69
70 - tokenizer = await self.engine_client.get_tokenizer()
71 - renderer = self._get_renderer(tokenizer)
72 -
70 if isinstance(request, TokenizeChatRequest):
71 tool_dicts = (
72 None
@@ -86,7 +83,7 @@ class OpenAIServingTokenization(OpenAIServing):
83
84 _, engine_prompts = await self._preprocess_chat(
85 request,
89 - tokenizer,
86 + self.renderer,
87 request.messages,
88 tool_dicts=tool_dicts,
89 chat_template=request.chat_template or self.chat_template,
@@ -97,6 +94,7 @@ class OpenAIServingTokenization(OpenAIServing):
94 add_special_tokens=request.add_special_tokens,
95 )
96 else:
97 + renderer = self._get_completion_renderer()
98 engine_prompts = await renderer.render_prompt(
99 prompt_or_prompts=request.prompt,
100 config=self._build_render_config(request),
@@ -116,6 +114,7 @@ class OpenAIServingTokenization(OpenAIServing):
114
115 token_strs = None
116 if request.return_token_strs:
117 + tokenizer = self.renderer.get_tokenizer()
118 token_strs = tokenizer.convert_ids_to_tokens(input_ids)
119
120 return TokenizeResponse(
@@ -137,8 +136,7 @@ class OpenAIServingTokenization(OpenAIServing):
136 request_id = f"tokenize-{self._base_request_id(raw_request)}"
137
138 lora_request = self._maybe_get_adapters(request)
140 -
141 - tokenizer = await self.engine_client.get_tokenizer()
139 + tokenizer = self.renderer.get_tokenizer()
140
141 self._log_inputs(
142 request_id,
@@ -161,7 +159,7 @@ class OpenAIServingTokenization(OpenAIServing):
159 ) -> TokenizerInfoResponse | ErrorResponse:
160 """Get comprehensive tokenizer information."""
161 try:
164 - tokenizer = await self.engine_client.get_tokenizer()
162 + tokenizer = self.renderer.get_tokenizer()
163 info = TokenizerInfo(tokenizer, self.chat_template).to_dict()
164 return TokenizerInfoResponse(**info)
165 except Exception as e:
vllm/entrypoints/utils.py
-43
@@ -6,7 +6,6 @@ import dataclasses
6 import functools
7 import os
8 from argparse import Namespace
9 -from pathlib import Path
9 from typing import TYPE_CHECKING, Any
10
11 import regex as re
@@ -14,17 +13,9 @@ from fastapi import Request
13 from fastapi.responses import JSONResponse, StreamingResponse
14 from starlette.background import BackgroundTask, BackgroundTasks
15
17 -from vllm.config import ModelConfig
16 from vllm.engine.arg_utils import EngineArgs
19 -from vllm.engine.protocol import EngineClient
20 -from vllm.entrypoints.chat_utils import (
21 - load_chat_template,
22 - resolve_hf_chat_template,
23 - resolve_mistral_chat_template,
24 -)
17 from vllm.logger import init_logger
18 from vllm.platforms import current_platform
27 -from vllm.tokenizers.mistral import MistralTokenizer
19 from vllm.utils.argparse_utils import FlexibleArgumentParser
20
21 if TYPE_CHECKING:
@@ -301,40 +292,6 @@ def process_lora_modules(
292 return lora_modules
293
294
304 -async def process_chat_template(
305 - args_chat_template: Path | str | None,
306 - engine_client: EngineClient,
307 - model_config: ModelConfig,
308 -) -> str | None:
309 - resolved_chat_template = load_chat_template(args_chat_template)
310 - if resolved_chat_template is not None:
311 - # Get the tokenizer to check official template
312 - tokenizer = await engine_client.get_tokenizer()
313 -
314 - if isinstance(tokenizer, MistralTokenizer):
315 - # The warning is logged in resolve_mistral_chat_template.
316 - resolved_chat_template = resolve_mistral_chat_template(
317 - chat_template=resolved_chat_template
318 - )
319 - else:
320 - hf_chat_template = resolve_hf_chat_template(
321 - tokenizer=tokenizer,
322 - chat_template=None,
323 - tools=None,
324 - model_config=model_config,
325 - )
326 -
327 - if hf_chat_template != resolved_chat_template:
328 - logger.warning(
329 - "Using supplied chat template: %s\n"
330 - "It is different from official chat template '%s'. "
331 - "This discrepancy may lead to performance degradation.",
332 - resolved_chat_template,
333 - model_config.model,
334 - )
335 - return resolved_chat_template
336 -
337 -
295 def sanitize_message(message: str) -> str:
296 # Avoid leaking memory address from object reprs
297 return re.sub(r" at 0x[0-9a-f]+>", ">", message)
vllm/inputs/preprocess.py
+7 -8
@@ -17,6 +17,7 @@ from vllm.multimodal.inputs import (
17 MultiModalUUIDDict,
18 )
19 from vllm.multimodal.processing import BaseMultiModalProcessor
20 +from vllm.renderers import renderer_from_config
21 from vllm.tokenizers import TokenizerLike
22 from vllm.utils.jsontree import json_iter_leaves
23 from vllm.v1.metrics.stats import MultiModalCacheStats
@@ -46,7 +47,6 @@ class InputPreprocessor:
47 def __init__(
48 self,
49 model_config: ModelConfig,
49 - tokenizer: TokenizerLike | None,
50 observability_config: ObservabilityConfig | None = None,
51 mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
52 mm_processor_cache: BaseMultiModalProcessorCache | None = None,
@@ -54,20 +54,19 @@ class InputPreprocessor:
54 super().__init__()
55
56 self.model_config = model_config
57 - self.tokenizer = tokenizer
57 self.observability_config = observability_config
58 + self.renderer = renderer_from_config(model_config)
59 self.mm_registry = mm_registry
60 self.mm_processor_cache = mm_processor_cache
61
62 self.mm_cache_stats = MultiModalCacheStats() if mm_processor_cache else None
63
64 - def get_tokenizer(self) -> TokenizerLike:
65 - if self.tokenizer is None:
66 - raise ValueError(
67 - "You cannot pass text prompts when `skip_tokenizer_init=True`"
68 - )
64 + @property
65 + def tokenizer(self) -> TokenizerLike | None:
66 + return self.renderer.tokenizer
67
70 - return self.tokenizer
68 + def get_tokenizer(self) -> TokenizerLike:
69 + return self.renderer.get_tokenizer()
70
71 def get_bos_token_id(self) -> int | None:
72 if self.tokenizer is None:
vllm/renderers/__init__.py new
+7
@@ -0,0 +1,7 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +from .protocol import RendererLike
5 +from .registry import RendererRegistry, renderer_from_config
6 +
7 +__all__ = ["RendererLike", "RendererRegistry", "renderer_from_config"]
vllm/renderers/deepseek_v32.py new
+119
@@ -0,0 +1,119 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +from typing import Any
4 +
5 +from vllm.config import ModelConfig
6 +from vllm.entrypoints.chat_utils import (
7 + ChatCompletionMessageParam,
8 + ConversationMessage,
9 + parse_chat_messages,
10 + parse_chat_messages_async,
11 +)
12 +from vllm.inputs import TextPrompt, TokensPrompt
13 +from vllm.logger import init_logger
14 +from vllm.tokenizers import cached_get_tokenizer
15 +from vllm.tokenizers.deepseek_v32 import DeepseekV32Tokenizer
16 +
17 +from .protocol import RendererLike
18 +
19 +logger = init_logger(__name__)
20 +
21 +
22 +class DeepseekV32Renderer(RendererLike):
23 + @classmethod
24 + def from_config(
25 + cls,
26 + config: ModelConfig,
27 + tokenizer_kwargs: dict[str, Any],
28 + ) -> "RendererLike":
29 + return cls(config, tokenizer_kwargs)
30 +
31 + def __init__(
32 + self,
33 + config: ModelConfig,
34 + tokenizer_kwargs: dict[str, Any],
35 + ) -> None:
36 + super().__init__()
37 +
38 + self.config = config
39 +
40 + if config.skip_tokenizer_init:
41 + tokenizer = None
42 + else:
43 + tokenizer = cached_get_tokenizer(
44 + tokenizer_cls=DeepseekV32Tokenizer,
45 + **tokenizer_kwargs,
46 + )
47 +
48 + self._tokenizer = tokenizer
49 +
50 + @property
51 + def tokenizer(self) -> DeepseekV32Tokenizer | None:
52 + return self._tokenizer
53 +
54 + def get_tokenizer(self) -> DeepseekV32Tokenizer:
55 + tokenizer = self.tokenizer
56 + if tokenizer is None:
57 + raise ValueError("Tokenizer not available when `skip_tokenizer_init=True`")
58 +
59 + return tokenizer
60 +
61 + def render_messages(
62 + self,
63 + messages: list[ChatCompletionMessageParam],
64 + **kwargs,
65 + ) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt]:
66 + tokenizer = self.get_tokenizer()
67 + conversation, mm_data, mm_uuids = parse_chat_messages(
68 + messages,
69 + self.config,
70 + content_format="string",
71 + )
72 +
73 + prompt_raw = tokenizer.apply_chat_template(
74 + conversation=conversation,
75 + messages=messages,
76 + **kwargs,
77 + )
78 +
79 + prompt = (
80 + TextPrompt(prompt=prompt_raw)
81 + if isinstance(prompt_raw, str)
82 + else TokensPrompt(prompt_token_ids=prompt_raw)
83 + )
84 + if mm_data is not None:
85 + prompt["multi_modal_data"] = mm_data
86 + if mm_uuids is not None:
87 + prompt["multi_modal_uuids"] = mm_uuids
88 +
89 + return conversation, prompt # type: ignore[return-value]
90 +
91 + async def render_messages_async(
92 + self,
93 + messages: list[ChatCompletionMessageParam],
94 + **kwargs,
95 + ) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt]:
96 + tokenizer = self.get_tokenizer()
97 + conversation, mm_data, mm_uuids = await parse_chat_messages_async(
98 + messages,
99 + self.config,
100 + content_format="string",
101 + )
102 +
103 + prompt_raw = tokenizer.apply_chat_template(
104 + conversation=conversation,
105 + messages=messages,
106 + **kwargs,
107 + )
108 +
109 + prompt = (
110 + TextPrompt(prompt=prompt_raw)
111 + if isinstance(prompt_raw, str)
112 + else TokensPrompt(prompt_token_ids=prompt_raw)
113 + )
114 + if mm_data is not None:
115 + prompt["multi_modal_data"] = mm_data
116 + if mm_uuids is not None:
117 + prompt["multi_modal_uuids"] = mm_uuids
118 +
119 + return conversation, prompt # type: ignore[return-value]
vllm/renderers/grok2.py new
+119
@@ -0,0 +1,119 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +from typing import Any
4 +
5 +from vllm.config import ModelConfig
6 +from vllm.entrypoints.chat_utils import (
7 + ChatCompletionMessageParam,
8 + ConversationMessage,
9 + parse_chat_messages,
10 + parse_chat_messages_async,
11 +)
12 +from vllm.inputs import TextPrompt, TokensPrompt
13 +from vllm.logger import init_logger
14 +from vllm.tokenizers import cached_get_tokenizer
15 +from vllm.tokenizers.grok2 import Grok2Tokenizer
16 +
17 +from .protocol import RendererLike
18 +
19 +logger = init_logger(__name__)
20 +
21 +
22 +class Grok2Renderer(RendererLike):
23 + @classmethod
24 + def from_config(
25 + cls,
26 + config: ModelConfig,
27 + tokenizer_kwargs: dict[str, Any],
28 + ) -> "RendererLike":
29 + return cls(config, tokenizer_kwargs)
30 +
31 + def __init__(
32 + self,
33 + config: ModelConfig,
34 + tokenizer_kwargs: dict[str, Any],
35 + ) -> None:
36 + super().__init__()
37 +
38 + self.config = config
39 +
40 + if config.skip_tokenizer_init:
41 + tokenizer = None
42 + else:
43 + tokenizer = cached_get_tokenizer(
44 + tokenizer_cls=Grok2Tokenizer,
45 + **tokenizer_kwargs,
46 + )
47 +
48 + self._tokenizer = tokenizer
49 +
50 + @property
51 + def tokenizer(self) -> Grok2Tokenizer | None:
52 + return self._tokenizer
53 +
54 + def get_tokenizer(self) -> Grok2Tokenizer:
55 + tokenizer = self.tokenizer
56 + if tokenizer is None:
57 + raise ValueError("Tokenizer not available when `skip_tokenizer_init=True`")
58 +
59 + return tokenizer
60 +
61 + def render_messages(
62 + self,
63 + messages: list[ChatCompletionMessageParam],
64 + **kwargs,
65 + ) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt]:
66 + tokenizer = self.get_tokenizer()
67 + conversation, mm_data, mm_uuids = parse_chat_messages(
68 + messages,
69 + self.config,
70 + content_format="string",
71 + )
72 +
73 + prompt_raw = tokenizer.apply_chat_template(
74 + conversation=conversation,
75 + messages=messages,
76 + **kwargs,
77 + )
78 +
79 + prompt = (
80 + TextPrompt(prompt=prompt_raw)
81 + if isinstance(prompt_raw, str)
82 + else TokensPrompt(prompt_token_ids=prompt_raw)
83 + )
84 + if mm_data is not None:
85 + prompt["multi_modal_data"] = mm_data
86 + if mm_uuids is not None:
87 + prompt["multi_modal_uuids"] = mm_uuids
88 +
89 + return conversation, prompt # type: ignore[return-value]
90 +
91 + async def render_messages_async(
92 + self,
93 + messages: list[ChatCompletionMessageParam],
94 + **kwargs,
95 + ) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt]:
96 + tokenizer = self.get_tokenizer()
97 + conversation, mm_data, mm_uuids = await parse_chat_messages_async(
98 + messages,
99 + self.config,
100 + content_format="string",
101 + )
102 +
103 + prompt_raw = tokenizer.apply_chat_template(
104 + conversation=conversation,
105 + messages=messages,
106 + **kwargs,
107 + )
108 +
109 + prompt = (
110 + TextPrompt(prompt=prompt_raw)
111 + if isinstance(prompt_raw, str)
112 + else TokensPrompt(prompt_token_ids=prompt_raw)
113 + )
114 + if mm_data is not None:
115 + prompt["multi_modal_data"] = mm_data
116 + if mm_uuids is not None:
117 + prompt["multi_modal_uuids"] = mm_uuids
118 +
119 + return conversation, prompt # type: ignore[return-value]
vllm/renderers/hf.py new
+600
@@ -0,0 +1,600 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +import inspect
4 +from collections import deque
5 +from collections.abc import Set
6 +from functools import lru_cache
7 +from typing import Any, cast
8 +
9 +import jinja2
10 +import jinja2.ext
11 +import jinja2.meta
12 +import jinja2.nodes
13 +import jinja2.parser
14 +import jinja2.sandbox
15 +
16 +from vllm.config import ModelConfig
17 +from vllm.entrypoints.chat_utils import (
18 + ChatCompletionMessageParam,
19 + ChatTemplateContentFormat,
20 + ChatTemplateContentFormatOption,
21 + ChatTemplateResolutionError,
22 + ConversationMessage,
23 + load_chat_template,
24 + parse_chat_messages,
25 + parse_chat_messages_async,
26 +)
27 +from vllm.inputs import TextPrompt, TokensPrompt
28 +from vllm.logger import init_logger
29 +from vllm.tokenizers import cached_get_tokenizer
30 +from vllm.tokenizers.hf import CachedHfTokenizer, HfTokenizer
31 +from vllm.transformers_utils.chat_templates import get_chat_template_fallback_path
32 +from vllm.transformers_utils.processor import cached_get_processor
33 +from vllm.utils.func_utils import supports_kw
34 +
35 +from .protocol import RendererLike
36 +
37 +logger = init_logger(__name__)
38 +
39 +
40 +_PROCESSOR_CHAT_TEMPLATES = dict[tuple[str, bool], str | None]()
41 +"""
42 +Used in `_try_get_processor_chat_template` to avoid calling
43 +`cached_get_processor` again if the processor fails to be loaded.
44 +
45 +This is needed because `lru_cache` does not cache when an exception happens.
46 +"""
47 +
48 +
49 +def _try_get_processor_chat_template(
50 + tokenizer: HfTokenizer,
51 + *,
52 + trust_remote_code: bool,
53 +) -> str | None:
54 + cache_key = (tokenizer.name_or_path, trust_remote_code)
55 + if cache_key in _PROCESSOR_CHAT_TEMPLATES:
56 + return _PROCESSOR_CHAT_TEMPLATES[cache_key]
57 +
58 + from transformers import (
59 + PreTrainedTokenizer,
60 + PreTrainedTokenizerFast,
61 + ProcessorMixin,
62 + )
63 +
64 + try:
65 + processor = cached_get_processor(
66 + tokenizer.name_or_path,
67 + processor_cls=(
68 + PreTrainedTokenizer,
69 + PreTrainedTokenizerFast,
70 + ProcessorMixin,
71 + ),
72 + trust_remote_code=trust_remote_code,
73 + )
74 + if (
75 + isinstance(processor, ProcessorMixin)
76 + and hasattr(processor, "chat_template")
77 + and (chat_template := processor.chat_template) is not None
78 + ):
79 + _PROCESSOR_CHAT_TEMPLATES[cache_key] = chat_template
80 + return chat_template
81 + except Exception:
82 + logger.debug(
83 + "Failed to load AutoProcessor chat template for %s",
84 + tokenizer.name_or_path,
85 + exc_info=True,
86 + )
87 +
88 + _PROCESSOR_CHAT_TEMPLATES[cache_key] = None
89 + return None
90 +
91 +
92 +def resolve_chat_template(
93 + tokenizer: HfTokenizer,
94 + chat_template: str | None,
95 + tools: list[dict[str, Any]] | None,
96 + *,
97 + model_config: "ModelConfig",
98 +) -> str | None:
99 + # 1st priority: The given chat template
100 + if chat_template is not None:
101 + return chat_template
102 +
103 + # 2nd priority: AutoProcessor chat template, unless tool calling is enabled
104 + if tools is None:
105 + chat_template = _try_get_processor_chat_template(
106 + tokenizer,
107 + trust_remote_code=model_config.trust_remote_code,
108 + )
109 + if chat_template is not None:
110 + return chat_template
111 +
112 + # 3rd priority: AutoTokenizer chat template
113 + try:
114 + return tokenizer.get_chat_template(chat_template, tools=tools)
115 + except Exception:
116 + logger.debug(
117 + "Failed to load AutoTokenizer chat template for %s",
118 + tokenizer.name_or_path,
119 + exc_info=True,
120 + )
121 +
122 + # 4th priority: Predefined fallbacks
123 + path = get_chat_template_fallback_path(
124 + model_type=model_config.hf_config.model_type,
125 + tokenizer_name_or_path=tokenizer.name_or_path,
126 + )
127 + if path is not None:
128 + logger.info_once(
129 + "Loading chat template fallback for %s as there isn't one "
130 + "defined on HF Hub.",
131 + tokenizer.name_or_path,
132 + )
133 + chat_template = load_chat_template(path)
134 + else:
135 + logger.debug_once(
136 + "There is no chat template fallback for %s", tokenizer.name_or_path
137 + )
138 +
139 + return chat_template
140 +
141 +
142 +def _is_var_access(node: jinja2.nodes.Node, varname: str) -> bool:
143 + if isinstance(node, jinja2.nodes.Name):
144 + return node.ctx == "load" and node.name == varname
145 +
146 + return False
147 +
148 +
149 +def _is_attr_access(node: jinja2.nodes.Node, varname: str, key: str) -> bool:
150 + if isinstance(node, jinja2.nodes.Getitem):
151 + return (
152 + _is_var_access(node.node, varname)
153 + and isinstance(node.arg, jinja2.nodes.Const)
154 + and node.arg.value == key
155 + )
156 +
157 + if isinstance(node, jinja2.nodes.Getattr):
158 + return _is_var_access(node.node, varname) and node.attr == key
159 +
160 + return False
161 +
162 +
163 +def _is_var_or_elems_access(
164 + node: jinja2.nodes.Node,
165 + varname: str,
166 + key: str | None = None,
167 +) -> bool:
168 + if isinstance(node, jinja2.nodes.Filter):
169 + return node.node is not None and _is_var_or_elems_access(
170 + node.node, varname, key
171 + )
172 + if isinstance(node, jinja2.nodes.Test):
173 + return _is_var_or_elems_access(node.node, varname, key)
174 +
175 + if isinstance(node, jinja2.nodes.Getitem) and isinstance(
176 + node.arg, jinja2.nodes.Slice
177 + ):
178 + return _is_var_or_elems_access(node.node, varname, key)
179 +
180 + return _is_attr_access(node, varname, key) if key else _is_var_access(node, varname)
181 +
182 +
183 +def _iter_nodes_assign_var_or_elems(root: jinja2.nodes.Node, varname: str):
184 + # Global variable that is implicitly defined at the root
185 + yield root, varname
186 +
187 + # Iterative BFS
188 + related_varnames = deque([varname])
189 + while related_varnames:
190 + related_varname = related_varnames.popleft()
191 +
192 + for assign_ast in root.find_all(jinja2.nodes.Assign):
193 + lhs = assign_ast.target
194 + rhs = assign_ast.node
195 +
196 + if _is_var_or_elems_access(rhs, related_varname):
197 + assert isinstance(lhs, jinja2.nodes.Name)
198 + yield assign_ast, lhs.name
199 +
200 + # Avoid infinite looping for self-assignment
201 + if lhs.name != related_varname:
202 + related_varnames.append(lhs.name)
203 +
204 +
205 +# NOTE: The proper way to handle this is to build a CFG so that we can handle
206 +# the scope in which each variable is defined, but that is too complicated
207 +def _iter_nodes_assign_messages_item(root: jinja2.nodes.Node):
208 + messages_varnames = [
209 + varname for _, varname in _iter_nodes_assign_var_or_elems(root, "messages")
210 + ]
211 +
212 + # Search for {%- for message in messages -%} loops
213 + for loop_ast in root.find_all(jinja2.nodes.For):
214 + loop_iter = loop_ast.iter
215 + loop_target = loop_ast.target
216 +
217 + for varname in messages_varnames:
218 + if _is_var_or_elems_access(loop_iter, varname):
219 + assert isinstance(loop_target, jinja2.nodes.Name)
220 + yield loop_ast, loop_target.name
221 + break
222 +
223 +
224 +def _iter_nodes_assign_content_item(root: jinja2.nodes.Node):
225 + message_varnames = [
226 + varname for _, varname in _iter_nodes_assign_messages_item(root)
227 + ]
228 +
229 + # Search for {%- for content in message['content'] -%} loops
230 + for loop_ast in root.find_all(jinja2.nodes.For):
231 + loop_iter = loop_ast.iter
232 + loop_target = loop_ast.target
233 +
234 + for varname in message_varnames:
235 + if _is_var_or_elems_access(loop_iter, varname, "content"):
236 + assert isinstance(loop_target, jinja2.nodes.Name)
237 + yield loop_ast, loop_target.name
238 + break
239 +
240 +
241 +def _try_extract_ast(chat_template: str) -> jinja2.nodes.Template | None:
242 + import transformers.utils.chat_template_utils as hf_chat_utils
243 +
244 + try:
245 + jinja_compiled = hf_chat_utils._compile_jinja_template(chat_template)
246 + return jinja_compiled.environment.parse(chat_template)
247 + except Exception:
248 + logger.exception("Error when compiling Jinja template")
249 + return None
250 +
251 +
252 +@lru_cache(maxsize=32)
253 +def _detect_content_format(
254 + chat_template: str,
255 + *,
256 + default: ChatTemplateContentFormat,
257 +) -> ChatTemplateContentFormat:
258 + jinja_ast = _try_extract_ast(chat_template)
259 + if jinja_ast is None:
260 + return default
261 +
262 + try:
263 + next(_iter_nodes_assign_content_item(jinja_ast))
264 + except StopIteration:
265 + return "string"
266 + except Exception:
267 + logger.exception("Error when parsing AST of Jinja template")
268 + return default
269 + else:
270 + return "openai"
271 +
272 +
273 +def _resolve_chat_template_content_format(
274 + chat_template: str | None,
275 + tools: list[dict[str, Any]] | None,
276 + tokenizer: HfTokenizer,
277 + *,
278 + model_config: "ModelConfig",
279 +) -> ChatTemplateContentFormat:
280 + resolved_chat_template = resolve_chat_template(
281 + tokenizer,
282 + chat_template=chat_template,
283 + tools=tools,
284 + model_config=model_config,
285 + )
286 +
287 + jinja_text = (
288 + resolved_chat_template
289 + if isinstance(resolved_chat_template, str)
290 + else load_chat_template(chat_template, is_literal=True)
291 + )
292 +
293 + detected_format = (
294 + "string"
295 + if jinja_text is None
296 + else _detect_content_format(jinja_text, default="string")
297 + )
298 +
299 + return detected_format
300 +
301 +
302 +@lru_cache
303 +def _log_chat_template_content_format(
304 + chat_template: str | None, # For caching purposes
305 + given_format: ChatTemplateContentFormatOption,
306 + detected_format: ChatTemplateContentFormatOption,
307 +):
308 + logger.info(
309 + "Detected the chat template content format to be '%s'. "
310 + "You can set `--chat-template-content-format` to override this.",
311 + detected_format,
312 + )
313 +
314 + if given_format != "auto" and given_format != detected_format:
315 + logger.warning(
316 + "You specified `--chat-template-content-format %s` "
317 + "which is different from the detected format '%s'. "
318 + "If our automatic detection is incorrect, please consider "
319 + "opening a GitHub issue so that we can improve it: "
320 + "https://github.com/vllm-project/vllm/issues/new/choose",
321 + given_format,
322 + detected_format,
323 + )
324 +
325 +
326 +def resolve_chat_template_content_format(
327 + chat_template: str | None,
328 + tools: list[dict[str, Any]] | None,
329 + given_format: ChatTemplateContentFormatOption,
330 + tokenizer: HfTokenizer,
331 + *,
332 + model_config: "ModelConfig",
333 +) -> ChatTemplateContentFormat:
334 + if given_format != "auto":
335 + return given_format
336 +
337 + detected_format = _resolve_chat_template_content_format(
338 + chat_template,
339 + tools,
340 + tokenizer,
341 + model_config=model_config,
342 + )
343 +
344 + _log_chat_template_content_format(
345 + chat_template,
346 + given_format=given_format,
347 + detected_format=detected_format,
348 + )
349 +
350 + return detected_format
351 +
352 +
353 +# adapted from https://github.com/huggingface/transformers/blob/v4.56.2/src/transformers/utils/chat_template_utils.py#L398-L412
354 +# only preserve the parse function used to resolve chat template kwargs
355 +class AssistantTracker(jinja2.ext.Extension):
356 + tags = {"generation"}
357 +
358 + def parse(self, parser: jinja2.parser.Parser) -> jinja2.nodes.Node:
359 + lineno = next(parser.stream).lineno
360 + body = parser.parse_statements(("name:endgeneration",), drop_needle=True)
361 + call = self.call_method("_generation_support")
362 + call_block = jinja2.nodes.CallBlock(call, [], [], body)
363 + return call_block.set_lineno(lineno)
364 +
365 +
366 +def _resolve_chat_template_kwargs(chat_template: str) -> Set[str]:
367 + env = jinja2.sandbox.ImmutableSandboxedEnvironment(
368 + trim_blocks=True,
369 + lstrip_blocks=True,
370 + extensions=[AssistantTracker, jinja2.ext.loopcontrols],
371 + )
372 + parsed_content = env.parse(chat_template)
373 + template_vars = jinja2.meta.find_undeclared_variables(parsed_content)
374 + return template_vars
375 +
376 +
377 +_cached_resolve_chat_template_kwargs = lru_cache(_resolve_chat_template_kwargs)
378 +
379 +
380 +@lru_cache
381 +def _get_hf_base_chat_template_params() -> frozenset[str]:
382 + from transformers import PreTrainedTokenizer
383 +
384 + # Get standard parameters from HuggingFace's base tokenizer class.
385 + # This dynamically extracts parameters from PreTrainedTokenizer's
386 + # apply_chat_template method, ensuring compatibility with tokenizers
387 + # that use **kwargs to receive standard parameters.
388 +
389 + # Read signature from HF's base class - the single source of truth
390 + base_sig = inspect.signature(PreTrainedTokenizer.apply_chat_template)
391 +
392 + # Exclude VAR_KEYWORD (**kwargs) and VAR_POSITIONAL (*args) placeholders
393 + return frozenset(
394 + p.name
395 + for p in base_sig.parameters.values()
396 + if p.kind
397 + not in (inspect.Parameter.VAR_KEYWORD, inspect.Parameter.VAR_POSITIONAL)
398 + )
399 +
400 +
401 +def resolve_chat_template_kwargs(
402 + tokenizer: HfTokenizer,
403 + chat_template: str,
404 + chat_template_kwargs: dict[str, Any],
405 + raise_on_unexpected: bool = True,
406 +) -> dict[str, Any]:
407 + # We exclude chat_template from kwargs here, because
408 + # chat template has been already resolved at this stage
409 + unexpected_vars = {"chat_template", "tokenize"}
410 + if raise_on_unexpected and (
411 + unexpected_in_kwargs := unexpected_vars & chat_template_kwargs.keys()
412 + ):
413 + raise ValueError(
414 + "Found unexpected chat template kwargs from request: "
415 + f"{unexpected_in_kwargs}"
416 + )
417 +
418 + fn_kw = {
419 + k
420 + for k in chat_template_kwargs
421 + if supports_kw(tokenizer.apply_chat_template, k, allow_var_kwargs=False)
422 + }
423 + template_vars = _cached_resolve_chat_template_kwargs(chat_template)
424 +
425 + # Allow standard HF parameters even if tokenizer uses **kwargs to receive them
426 + hf_base_params = _get_hf_base_chat_template_params()
427 +
428 + accept_vars = (fn_kw | template_vars | hf_base_params) - unexpected_vars
429 + return {k: v for k, v in chat_template_kwargs.items() if k in accept_vars}
430 +
431 +
432 +def safe_apply_chat_template(
433 + model_config: "ModelConfig",
434 + tokenizer: HfTokenizer,
435 + conversation: list[ConversationMessage],
436 + *,
437 + tools: list[dict[str, Any]] | None = None,
438 + chat_template: str | None = None,
439 + tokenize: bool = True,
440 + **kwargs,
441 +) -> str | list[int]:
442 + chat_template = resolve_chat_template(
443 + tokenizer,
444 + chat_template=chat_template,
445 + tools=tools,
446 + model_config=model_config,
447 + )
448 + if chat_template is None:
449 + raise ChatTemplateResolutionError(
450 + "As of transformers v4.44, default chat template is no longer "
451 + "allowed, so you must provide a chat template if the tokenizer "
452 + "does not define one."
453 + )
454 +
455 + resolved_kwargs = resolve_chat_template_kwargs(
456 + tokenizer=tokenizer,
457 + chat_template=chat_template,
458 + chat_template_kwargs=kwargs,
459 + )
460 +
461 + try:
462 + return tokenizer.apply_chat_template(
463 + conversation=conversation, # type: ignore[arg-type]
464 + tools=tools, # type: ignore[arg-type]
465 + chat_template=chat_template,
466 + tokenize=tokenize,
467 + **resolved_kwargs,
468 + )
469 + # External library exceptions can sometimes occur despite the framework's
470 + # internal exception management capabilities.
471 + except Exception as e:
472 + # Log and report any library-related exceptions for further
473 + # investigation.
474 + logger.exception(
475 + "An error occurred in `transformers` while applying chat template"
476 + )
477 + raise ValueError(str(e)) from e
478 +
479 +
480 +class HfRenderer(RendererLike):
481 + @classmethod
482 + def from_config(
483 + cls,
484 + config: ModelConfig,
485 + tokenizer_kwargs: dict[str, Any],
486 + ) -> "RendererLike":
487 + return cls(config, tokenizer_kwargs)
488 +
489 + def __init__(
490 + self,
491 + config: ModelConfig,
492 + tokenizer_kwargs: dict[str, Any],
493 + ) -> None:
494 + super().__init__()
495 +
496 + self.config = config
497 +
498 + if config.skip_tokenizer_init:
499 + tokenizer = None
500 + else:
501 + tokenizer = cast(
502 + HfTokenizer,
503 + cached_get_tokenizer(
504 + tokenizer_cls=CachedHfTokenizer, # type: ignore[type-abstract]
505 + **tokenizer_kwargs,
506 + ),
507 + )
508 +
509 + self._tokenizer = tokenizer
510 +
511 + @property
512 + def tokenizer(self) -> HfTokenizer | None:
513 + return self._tokenizer
514 +
515 + def get_tokenizer(self) -> HfTokenizer:
516 + tokenizer = self.tokenizer
517 + if tokenizer is None:
518 + raise ValueError("Tokenizer not available when `skip_tokenizer_init=True`")
519 +
520 + return tokenizer
521 +
522 + def render_messages(
523 + self,
524 + messages: list[ChatCompletionMessageParam],
525 + chat_template_content_format: ChatTemplateContentFormatOption = "auto",
526 + **kwargs,
527 + ) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt]:
528 + model_config = self.config
529 + tokenizer = self.get_tokenizer()
530 +
531 + conversation, mm_data, mm_uuids = parse_chat_messages(
532 + messages,
533 + model_config,
534 + content_format=resolve_chat_template_content_format(
535 + chat_template=kwargs.get("chat_template"),
536 + tools=kwargs.get("tools"),
537 + given_format=chat_template_content_format,
538 + tokenizer=tokenizer,
539 + model_config=model_config,
540 + ),
541 + )
542 +
543 + prompt_raw = safe_apply_chat_template(
544 + model_config,
545 + tokenizer,
546 + conversation,
547 + **kwargs,
548 + )
549 +
550 + prompt = (
551 + TextPrompt(prompt=prompt_raw)
552 + if isinstance(prompt_raw, str)
553 + else TokensPrompt(prompt_token_ids=prompt_raw)
554 + )
555 + if mm_data is not None:
556 + prompt["multi_modal_data"] = mm_data
557 + if mm_uuids is not None:
558 + prompt["multi_modal_uuids"] = mm_uuids
559 +
560 + return conversation, prompt # type: ignore[return-value]
561 +
562 + async def render_messages_async(
563 + self,
564 + messages: list[ChatCompletionMessageParam],
565 + chat_template_content_format: ChatTemplateContentFormatOption = "auto",
566 + **kwargs,
567 + ) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt]:
568 + model_config = self.config
569 + tokenizer = self.get_tokenizer()
570 +
571 + conversation, mm_data, mm_uuids = await parse_chat_messages_async(
572 + messages,
573 + model_config,
574 + content_format=resolve_chat_template_content_format(
575 + chat_template=kwargs.get("chat_template"),
576 + tools=kwargs.get("tools"),
577 + given_format=chat_template_content_format,
578 + tokenizer=tokenizer,
579 + model_config=model_config,
580 + ),
581 + )
582 +
583 + prompt_raw = safe_apply_chat_template(
584 + model_config,
585 + tokenizer,
586 + conversation,
587 + **kwargs,
588 + )
589 +
590 + prompt = (
591 + TextPrompt(prompt=prompt_raw)
592 + if isinstance(prompt_raw, str)
593 + else TokensPrompt(prompt_token_ids=prompt_raw)
594 + )
595 + if mm_data is not None:
596 + prompt["multi_modal_data"] = mm_data
597 + if mm_uuids is not None:
598 + prompt["multi_modal_uuids"] = mm_uuids
599 +
600 + return conversation, prompt # type: ignore[return-value]
vllm/renderers/mistral.py new
+147
@@ -0,0 +1,147 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +from concurrent.futures import ThreadPoolExecutor
4 +from typing import Any
5 +
6 +from vllm.config import ModelConfig
7 +from vllm.entrypoints.chat_utils import (
8 + ChatCompletionMessageParam,
9 + ConversationMessage,
10 + parse_chat_messages,
11 + parse_chat_messages_async,
12 +)
13 +from vllm.inputs import TextPrompt, TokensPrompt
14 +from vllm.logger import init_logger
15 +from vllm.tokenizers import cached_get_tokenizer
16 +from vllm.tokenizers.mistral import MistralTokenizer
17 +from vllm.utils.async_utils import make_async
18 +
19 +from .protocol import RendererLike
20 +
21 +logger = init_logger(__name__)
22 +
23 +
24 +def safe_apply_chat_template(
25 + tokenizer: MistralTokenizer,
26 + messages: list[ChatCompletionMessageParam],
27 + **kwargs,
28 +) -> str | list[int]:
29 + from mistral_common.exceptions import MistralCommonException
30 +
31 + try:
32 + return tokenizer.apply_chat_template(messages, **kwargs)
33 + # mistral-common uses assert statements to stop processing of input
34 + # if input does not comply with the expected format.
35 + # We convert those assertion errors to ValueErrors so they can be
36 + # properly caught in the preprocessing_input step
37 + except (AssertionError, MistralCommonException) as e:
38 + raise ValueError(str(e)) from e
39 +
40 + # External library exceptions can sometimes occur despite the framework's
41 + # internal exception management capabilities.
42 + except Exception as e:
43 + # Log and report any library-related exceptions for further
44 + # investigation.
45 + logger.exception(
46 + "An error occurred in `mistral_common` while applying chat template"
47 + )
48 + raise ValueError(str(e)) from e
49 +
50 +
51 +class MistralRenderer(RendererLike):
52 + @classmethod
53 + def from_config(
54 + cls,
55 + config: ModelConfig,
56 + tokenizer_kwargs: dict[str, Any],
57 + ) -> "RendererLike":
58 + return cls(config, tokenizer_kwargs)
59 +
60 + def __init__(
61 + self,
62 + config: ModelConfig,
63 + tokenizer_kwargs: dict[str, Any],
64 + ) -> None:
65 + super().__init__()
66 +
67 + self.config = config
68 +
69 + if config.skip_tokenizer_init:
70 + tokenizer = None
71 + else:
72 + tokenizer = cached_get_tokenizer(
73 + tokenizer_cls=MistralTokenizer,
74 + **tokenizer_kwargs,
75 + )
76 +
77 + self._tokenizer = tokenizer
78 +
79 + self._apply_chat_template_executor = ThreadPoolExecutor(max_workers=1)
80 + self._apply_chat_template_async = make_async(
81 + safe_apply_chat_template, executor=self._apply_chat_template_executor
82 + )
83 +
84 + @property
85 + def tokenizer(self) -> MistralTokenizer | None:
86 + return self._tokenizer
87 +
88 + def get_tokenizer(self) -> MistralTokenizer:
89 + tokenizer = self.tokenizer
90 + if tokenizer is None:
91 + raise ValueError("Tokenizer not available when `skip_tokenizer_init=True`")
92 +
93 + return tokenizer
94 +
95 + def render_messages(
96 + self,
97 + messages: list[ChatCompletionMessageParam],
98 + **kwargs,
99 + ) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt]:
100 + tokenizer = self.get_tokenizer()
101 + conversation, mm_data, mm_uuids = parse_chat_messages(
102 + messages,
103 + self.config,
104 + content_format="string",
105 + )
106 +
107 + prompt_raw = safe_apply_chat_template(tokenizer, messages, **kwargs)
108 +
109 + prompt = (
110 + TextPrompt(prompt=prompt_raw)
111 + if isinstance(prompt_raw, str)
112 + else TokensPrompt(prompt_token_ids=prompt_raw)
113 + )
114 + if mm_data is not None:
115 + prompt["multi_modal_data"] = mm_data
116 + if mm_uuids is not None:
117 + prompt["multi_modal_uuids"] = mm_uuids
118 +
119 + return conversation, prompt # type: ignore[return-value]
120 +
121 + async def render_messages_async(
122 + self,
123 + messages: list[ChatCompletionMessageParam],
124 + **kwargs,
125 + ) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt]:
126 + tokenizer = self.get_tokenizer()
127 + conversation, mm_data, mm_uuids = await parse_chat_messages_async(
128 + messages,
129 + self.config,
130 + content_format="string",
131 + )
132 +
133 + prompt_raw = await self._apply_chat_template_async(
134 + tokenizer, messages, **kwargs
135 + )
136 +
137 + prompt = (
138 + TextPrompt(prompt=prompt_raw)
139 + if isinstance(prompt_raw, str)
140 + else TokensPrompt(prompt_token_ids=prompt_raw)
141 + )
142 + if mm_data is not None:
143 + prompt["multi_modal_data"] = mm_data
144 + if mm_uuids is not None:
145 + prompt["multi_modal_uuids"] = mm_uuids
146 +
147 + return conversation, prompt # type: ignore[return-value]
vllm/renderers/protocol.py new
+48
@@ -0,0 +1,48 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +from typing import TYPE_CHECKING, Any, Protocol
4 +
5 +from vllm.inputs import TextPrompt, TokensPrompt
6 +from vllm.tokenizers import TokenizerLike
7 +
8 +if TYPE_CHECKING:
9 + from vllm.config import ModelConfig
10 + from vllm.entrypoints.chat_utils import (
11 + ChatCompletionMessageParam,
12 + ConversationMessage,
13 + )
14 +
15 +
16 +class RendererLike(Protocol):
17 + @classmethod
18 + def from_config(
19 + cls,
20 + config: "ModelConfig",
21 + tokenizer_kwargs: dict[str, Any],
22 + ) -> "RendererLike":
23 + raise NotImplementedError
24 +
25 + @property
26 + def tokenizer(self) -> TokenizerLike | None:
27 + raise NotImplementedError
28 +
29 + def get_tokenizer(self) -> TokenizerLike:
30 + tokenizer = self.tokenizer
31 + if tokenizer is None:
32 + raise ValueError("Tokenizer not available when `skip_tokenizer_init=True`")
33 +
34 + return tokenizer
35 +
36 + def render_messages(
37 + self,
38 + messages: list["ChatCompletionMessageParam"],
39 + **kwargs,
40 + ) -> tuple[list["ConversationMessage"], TextPrompt | TokensPrompt]:
41 + raise NotImplementedError
42 +
43 + async def render_messages_async(
44 + self,
45 + messages: list["ChatCompletionMessageParam"],
46 + **kwargs,
47 + ) -> tuple[list["ConversationMessage"], TextPrompt | TokensPrompt]:
48 + return self.render_messages(messages, **kwargs)
vllm/renderers/registry.py new
+88
@@ -0,0 +1,88 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +from dataclasses import dataclass, field
4 +from typing import TYPE_CHECKING, Any
5 +
6 +from vllm.logger import init_logger
7 +from vllm.tokenizers.registry import tokenizer_args_from_config
8 +from vllm.utils.import_utils import resolve_obj_by_qualname
9 +
10 +from .protocol import RendererLike
11 +
12 +if TYPE_CHECKING:
13 + from vllm.config import ModelConfig
14 +
15 +logger = init_logger(__name__)
16 +
17 +
18 +_VLLM_RENDERERS = {
19 + "deepseek_v32": ("deepseek_v32", "DeepseekV32Renderer"),
20 + "hf": ("hf", "HfRenderer"),
21 + "grok2": ("grok2", "Grok2Renderer"),
22 + "mistral": ("mistral", "MistralRenderer"),
23 + "terratorch": ("terratorch", "TerratorchRenderer"),
24 +}
25 +
26 +
27 +@dataclass
28 +class RendererRegistry:
29 + # Renderer mode -> (renderer module, renderer class)
30 + renderers: dict[str, tuple[str, str]] = field(default_factory=dict)
31 +
32 + def register(self, renderer_mode: str, module: str, class_name: str) -> None:
33 + if renderer_mode in self.renderers:
34 + logger.warning(
35 + "%s.%s is already registered for renderer_mode=%r. "
36 + "It is overwritten by the new one.",
37 + module,
38 + class_name,
39 + renderer_mode,
40 + )
41 +
42 + self.renderers[renderer_mode] = (module, class_name)
43 +
44 + return None
45 +
46 + def load_renderer_cls(self, renderer_mode: str) -> type[RendererLike]:
47 + if renderer_mode not in self.renderers:
48 + raise ValueError(f"No renderer registered for {renderer_mode=!r}.")
49 +
50 + module, class_name = self.renderers[renderer_mode]
51 + logger.debug_once(f"Loading {class_name} for {renderer_mode=!r}")
52 +
53 + return resolve_obj_by_qualname(f"{module}.{class_name}")
54 +
55 + def load_renderer(
56 + self,
57 + renderer_mode: str,
58 + config: "ModelConfig",
59 + tokenizer_kwargs: dict[str, Any],
60 + ) -> RendererLike:
61 + renderer_cls = self.load_renderer_cls(renderer_mode)
62 + return renderer_cls.from_config(config, tokenizer_kwargs)
63 +
64 +
65 +RENDERER_REGISTRY = RendererRegistry(
66 + {
67 + mode: (f"vllm.renderers.{mod_relname}", cls_name)
68 + for mode, (mod_relname, cls_name) in _VLLM_RENDERERS.items()
69 + }
70 +)
71 +"""The global `RendererRegistry` instance."""
72 +
73 +
74 +def renderer_from_config(config: "ModelConfig", **kwargs):
75 + tokenizer_mode, tokenizer_name, args, kwargs = tokenizer_args_from_config(
76 + config, **kwargs
77 + )
78 +
79 + if config.tokenizer_mode == "auto" and config.model_impl == "terratorch":
80 + renderer_mode = "terratorch"
81 + else:
82 + renderer_mode = tokenizer_mode
83 +
84 + return RENDERER_REGISTRY.load_renderer(
85 + renderer_mode,
86 + config,
87 + tokenizer_kwargs={**kwargs, "tokenizer_name": tokenizer_name},
88 + )
vllm/renderers/terratorch.py new
+85
@@ -0,0 +1,85 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +from typing import Any
4 +
5 +from vllm.config import ModelConfig
6 +from vllm.entrypoints.chat_utils import (
7 + ChatCompletionMessageParam,
8 + ConversationMessage,
9 + parse_chat_messages,
10 + parse_chat_messages_async,
11 +)
12 +from vllm.inputs import TextPrompt, TokensPrompt
13 +from vllm.logger import init_logger
14 +from vllm.tokenizers import TokenizerLike
15 +
16 +from .protocol import RendererLike
17 +
18 +logger = init_logger(__name__)
19 +
20 +
21 +class TerratorchRenderer(RendererLike):
22 + @classmethod
23 + def from_config(
24 + cls,
25 + config: "ModelConfig",
26 + tokenizer_kwargs: dict[str, Any],
27 + ) -> "RendererLike":
28 + return cls(config)
29 +
30 + def __init__(self, config: ModelConfig) -> None:
31 + super().__init__()
32 +
33 + self.config = config
34 +
35 + if not config.skip_tokenizer_init:
36 + raise ValueError("Terratorch renderer requires `skip_tokenizer_init=True`")
37 +
38 + @property
39 + def tokenizer(self) -> TokenizerLike | None:
40 + return None
41 +
42 + def get_tokenizer(self) -> TokenizerLike:
43 + raise ValueError("Tokenizer not available for Terratorch renderer")
44 +
45 + def render_messages(
46 + self,
47 + messages: list[ChatCompletionMessageParam],
48 + **kwargs,
49 + ) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt]:
50 + model_config = self.config
51 +
52 + conversation, mm_data, mm_uuids = parse_chat_messages(
53 + messages,
54 + model_config,
55 + content_format="string",
56 + )
57 +
58 + prompt = TokensPrompt(prompt_token_ids=[1])
59 + if mm_data is not None:
60 + prompt["multi_modal_data"] = mm_data
61 + if mm_uuids is not None:
62 + prompt["multi_modal_uuids"] = mm_uuids
63 +
64 + return conversation, prompt
65 +
66 + async def render_messages_async(
67 + self,
68 + messages: list[ChatCompletionMessageParam],
69 + **kwargs,
70 + ) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt]:
71 + model_config = self.config
72 +
73 + conversation, mm_data, mm_uuids = await parse_chat_messages_async(
74 + messages,
75 + model_config,
76 + content_format="string",
77 + )
78 +
79 + prompt = TokensPrompt(prompt_token_ids=[1]) # Dummy token IDs
80 + if mm_data is not None:
81 + prompt["multi_modal_data"] = mm_data
82 + if mm_uuids is not None:
83 + prompt["multi_modal_uuids"] = mm_uuids
84 +
85 + return conversation, prompt
vllm/v1/engine/async_llm.py
+8 -10
@@ -23,9 +23,10 @@ from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
23 from vllm.outputs import PoolingRequestOutput, RequestOutput
24 from vllm.plugins.io_processors import get_io_processor
25 from vllm.pooling_params import PoolingParams
26 +from vllm.renderers import RendererLike
27 from vllm.sampling_params import SamplingParams
28 from vllm.tasks import SupportedTask
28 -from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config
29 +from vllm.tokenizers import TokenizerLike
30 from vllm.tracing import init_tracer
31 from vllm.transformers_utils.config import maybe_register_config_serialize_by_value
32 from vllm.usage.usage_lib import UsageContext
@@ -106,9 +107,7 @@ class AsyncLLM(EngineClient):
107 "enabling logging without default stat loggers."
108 )
109
109 - tokenizer = cached_tokenizer_from_config(self.model_config)
110 -
111 - self.input_processor = InputProcessor(self.vllm_config, tokenizer)
110 + self.input_processor = InputProcessor(self.vllm_config)
111 self.io_processor = get_io_processor(
112 self.vllm_config,
113 self.model_config.io_processor_plugin,
@@ -709,13 +708,12 @@ class AsyncLLM(EngineClient):
708 def tokenizer(self) -> TokenizerLike | None:
709 return self.input_processor.tokenizer
710
712 - async def get_tokenizer(self) -> TokenizerLike:
713 - if self.tokenizer is None:
714 - raise ValueError(
715 - "Unable to get tokenizer because `skip_tokenizer_init=True`"
716 - )
711 + def get_tokenizer(self) -> TokenizerLike:
712 + return self.input_processor.get_tokenizer()
713
718 - return self.tokenizer
714 + @property
715 + def renderer(self) -> RendererLike:
716 + return self.input_processor.renderer
717
718 async def is_tracing_enabled(self) -> bool:
719 return self.observability_config.otlp_traces_endpoint is not None # type: ignore
vllm/v1/engine/input_processor.py
+9 -3
@@ -19,6 +19,7 @@ from vllm.multimodal.parse import MultiModalDataParser
19 from vllm.multimodal.processing.context import set_request_id
20 from vllm.multimodal.utils import argsort_mm_positions
21 from vllm.pooling_params import PoolingParams
22 +from vllm.renderers import RendererLike
23 from vllm.sampling_params import _SAMPLING_EPS, SamplingParams
24 from vllm.tokenizers import TokenizerLike
25 from vllm.tokenizers.mistral import MistralTokenizer
@@ -45,7 +46,6 @@ class InputProcessor:
46 def __init__(
47 self,
48 vllm_config: VllmConfig,
48 - tokenizer: TokenizerLike | None,
49 mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
50 ) -> None:
51 self.vllm_config = vllm_config
@@ -61,8 +61,7 @@ class InputProcessor:
61
62 self.input_preprocessor = InputPreprocessor(
63 self.model_config,
64 - tokenizer,
65 - self.vllm_config.observability_config,
64 + vllm_config.observability_config,
65 mm_registry,
66 mm_processor_cache=self.mm_processor_cache,
67 )
@@ -71,6 +70,13 @@ class InputProcessor:
70 def tokenizer(self) -> TokenizerLike | None:
71 return self.input_preprocessor.tokenizer
72
73 + def get_tokenizer(self) -> TokenizerLike:
74 + return self.input_preprocessor.get_tokenizer()
75 +
76 + @property
77 + def renderer(self) -> RendererLike:
78 + return self.input_preprocessor.renderer
79 +
80 def _validate_logprobs(
81 self,
82 params: SamplingParams,
vllm/v1/engine/llm_engine.py
+7 -9
@@ -21,9 +21,10 @@ from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
21 from vllm.outputs import PoolingRequestOutput, RequestOutput
22 from vllm.plugins.io_processors import get_io_processor
23 from vllm.pooling_params import PoolingParams
24 +from vllm.renderers import RendererLike
25 from vllm.sampling_params import SamplingParams
26 from vllm.tasks import SupportedTask
26 -from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config
27 +from vllm.tokenizers import TokenizerLike
28 from vllm.tracing import init_tracer
29 from vllm.usage.usage_lib import UsageContext
30 from vllm.v1.engine import EngineCoreRequest
@@ -84,9 +85,7 @@ class LLMEngine:
85 self.dp_group = None
86 self.should_execute_dummy_batch = False
87
87 - tokenizer = cached_tokenizer_from_config(self.model_config)
88 -
89 - self.input_processor = InputProcessor(self.vllm_config, tokenizer)
88 + self.input_processor = InputProcessor(self.vllm_config)
89 self.io_processor = get_io_processor(
90 self.vllm_config,
91 self.model_config.io_processor_plugin,
@@ -357,12 +356,11 @@ class LLMEngine:
356 return self.input_processor.tokenizer
357
358 def get_tokenizer(self) -> TokenizerLike:
360 - if self.tokenizer is None:
361 - raise ValueError(
362 - "Unable to get tokenizer because `skip_tokenizer_init=True`"
363 - )
359 + return self.input_processor.get_tokenizer()
360
365 - return self.tokenizer
361 + @property
362 + def renderer(self) -> RendererLike:
363 + return self.input_processor.renderer
364
365 def do_log_stats(self) -> None:
366 """Log stats if logging is enabled."""