[Frontend] Split ServingRender into renderer and entrypoint. (#44285)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
wang.yuqi committed
Jun 23, 2026 at 19:19 UTC
31ca9504b14c328291fda1ad1a2659180ae8eb34
26 files changed
+1095
-879
tests/entrypoints/openai/chat_completion/test_chat_error.py
+46
-13
@@ -17,9 +17,11 @@ from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
17
from vllm.entrypoints.openai.engine.protocol import GenerationError
18
from vllm.entrypoints.openai.models.protocol import BaseModelPath
19
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
20
-from vllm.entrypoints.serve.render.serving import OpenAIServingRender
20
+from vllm.entrypoints.serve.render.serving import ServingRender
21
from vllm.outputs import CompletionOutput, RequestOutput
22
from vllm.renderers.hf import HfRenderer
23
+from vllm.renderers.online_derenderer import OnlineDerenderer
24
+from vllm.renderers.online_renderer import OnlineRenderer
25
from vllm.tokenizers.registry import cached_tokenizer_from_config
26
from vllm.v1.engine.async_llm import AsyncLLM
27
@@ -88,19 +90,19 @@ def _build_serving_chat(engine: AsyncLLM) -> OpenAIServingChat:
90
engine_client=engine,
91
base_model_paths=BASE_MODEL_PATHS,
92
)
91
- serving_render = OpenAIServingRender(
93
+ online_renderer = OnlineRenderer(
94
model_config=engine.model_config,
95
renderer=engine.renderer,
94
- model_registry=models.registry,
96
request_logger=None,
97
chat_template=None,
98
chat_template_content_format="auto",
99
)
100
+
101
serving_chat = OpenAIServingChat(
102
engine,
103
models,
104
response_role="assistant",
103
- openai_serving_render=serving_render,
105
+ online_renderer=online_renderer,
106
request_logger=None,
107
chat_template=None,
108
chat_template_content_format="auto",
@@ -113,7 +115,7 @@ def _build_serving_chat(engine: AsyncLLM) -> OpenAIServingChat:
115
[{"prompt_token_ids": [1, 2, 3]}],
116
)
117
116
- serving_chat.openai_serving_render.preprocess_chat = AsyncMock(
118
+ serving_chat.online_renderer.preprocess_chat = AsyncMock(
119
side_effect=_fake_preprocess_chat
120
)
121
return serving_chat
@@ -187,13 +189,46 @@ async def test_openai_chat_keeps_mm_cache_for_engine_execution():
189
190
assert isinstance(result, tuple)
191
assert (
190
- serving_chat.openai_serving_render.preprocess_chat.call_args.kwargs[
191
- "skip_mm_cache"
192
- ]
192
+ serving_chat.online_renderer.preprocess_chat.call_args.kwargs["skip_mm_cache"]
193
is False
194
)
195
196
197
+def _build_serving_render(engine: AsyncLLM) -> ServingRender:
198
+ models = OpenAIServingModels(
199
+ engine_client=engine,
200
+ base_model_paths=BASE_MODEL_PATHS,
201
+ )
202
+ online_renderer = OnlineRenderer(
203
+ model_config=engine.model_config,
204
+ renderer=engine.renderer,
205
+ request_logger=None,
206
+ chat_template=None,
207
+ chat_template_content_format="auto",
208
+ )
209
+ online_derenderer = OnlineDerenderer(
210
+ model_config=engine.model_config,
211
+ renderer=engine.renderer,
212
+ request_logger=None,
213
+ chat_template=None,
214
+ chat_template_content_format="auto",
215
+ )
216
+
217
+ serving_render = ServingRender(models, online_renderer, online_derenderer)
218
+
219
+ async def _fake_preprocess_chat(*args, **kwargs):
220
+ # return conversation, engine_inputs
221
+ return (
222
+ [{"role": "user", "content": "Test"}],
223
+ [{"prompt_token_ids": [1, 2, 3]}],
224
+ )
225
+
226
+ serving_render.online_renderer.preprocess_chat = AsyncMock(
227
+ side_effect=_fake_preprocess_chat
228
+ )
229
+ return serving_render
230
+
231
+
232
@pytest.mark.asyncio
233
async def test_renderer_only_chat_request_skips_mm_cache():
234
mock_engine = MagicMock(spec=AsyncLLM)
@@ -202,20 +237,18 @@ async def test_renderer_only_chat_request_skips_mm_cache():
237
mock_engine.input_processor = MagicMock()
238
mock_engine.renderer = _build_renderer(mock_engine.model_config)
239
205
- serving_chat = _build_serving_chat(mock_engine)
240
+ serving_render = _build_serving_render(mock_engine)
241
242
request = ChatCompletionRequest(
243
model=MODEL_NAME,
244
messages=[{"role": "user", "content": "Test prompt"}],
245
)
246
212
- result = await serving_chat.openai_serving_render.render_chat_request(request)
247
+ result = await serving_render.render_chat_request(request)
248
249
assert result.token_ids == [1, 2, 3]
250
assert (
216
- serving_chat.openai_serving_render.preprocess_chat.call_args.kwargs[
217
- "skip_mm_cache"
218
- ]
251
+ serving_render.online_renderer.preprocess_chat.call_args.kwargs["skip_mm_cache"]
252
is True
253
)
254
tests/entrypoints/openai/chat_completion/test_serving_chat.py
+30
-47
@@ -38,7 +38,6 @@ from vllm.entrypoints.openai.models.serving import (
38
OpenAIServingModels,
39
)
40
from vllm.entrypoints.openai.parser.harmony_utils import get_encoding
41
-from vllm.entrypoints.serve.render.serving import OpenAIServingRender
41
from vllm.exceptions import VLLMValidationError
42
from vllm.inputs import TokensPrompt
43
from vllm.multimodal.inputs import PlaceholderRange
@@ -46,6 +45,7 @@ from vllm.outputs import CompletionOutput, RequestOutput
45
from vllm.parser import HarmonyParser
46
from vllm.renderers.hf import HfRenderer
47
from vllm.renderers.mistral import MistralRenderer
48
+from vllm.renderers.online_renderer import OnlineRenderer
49
from vllm.tokenizers import get_tokenizer
50
from vllm.tokenizers.mistral import MistralTokenizer
51
from vllm.tokenizers.registry import cached_tokenizer_from_config
@@ -567,13 +567,12 @@ def _build_renderer(model_config: MockModelConfig):
567
)
568
569
570
-def _build_serving_render(
570
+def _build_online_renderer(
571
engine, model_registry: OpenAIModelRegistry
572
-) -> OpenAIServingRender:
573
- return OpenAIServingRender(
572
+) -> OnlineRenderer:
573
+ return OnlineRenderer(
574
model_config=engine.model_config,
575
renderer=engine.renderer,
576
- model_registry=model_registry,
576
request_logger=None,
577
chat_template=CHAT_TEMPLATE,
578
chat_template_content_format="auto",
@@ -591,13 +590,13 @@ def _build_serving_chat(
590
engine_client=engine,
591
base_model_paths=BASE_MODEL_PATHS,
592
)
594
- openai_serving_render = _build_serving_render(engine, models.registry)
593
+ online_renderer = _build_online_renderer(engine, models.registry)
594
595
serving_chat = OpenAIServingChat(
596
engine,
597
models,
598
response_role="assistant",
600
- openai_serving_render=openai_serving_render,
599
+ online_renderer=online_renderer,
600
chat_template=CHAT_TEMPLATE,
601
chat_template_content_format="auto",
602
request_logger=None,
@@ -621,13 +620,13 @@ async def _async_serving_chat_init():
620
engine = MockEngine()
621
622
models = OpenAIServingModels(engine, BASE_MODEL_PATHS)
624
- openai_serving_render = _build_serving_render(engine, models.registry)
623
+ online_renderer = _build_online_renderer(engine, models.registry)
624
625
serving_completion = OpenAIServingChat(
626
engine,
627
models,
628
response_role="assistant",
630
- openai_serving_render=openai_serving_render,
629
+ online_renderer=online_renderer,
630
chat_template=CHAT_TEMPLATE,
631
chat_template_content_format="auto",
632
request_logger=None,
@@ -1396,9 +1395,7 @@ class TestServingChatWithHarmony:
1395
messages=messages,
1396
include_reasoning=include_reasoning,
1397
)
1399
- input_messages, _ = (
1400
- serving_chat.openai_serving_render._make_request_with_harmony(req)
1401
- )
1398
+ input_messages, _ = serving_chat.online_renderer._make_request_with_harmony(req)
1399
verify_harmony_messages(
1400
input_messages,
1401
[
@@ -1429,8 +1426,8 @@ class TestServingChatWithHarmony:
1426
1427
# Test the Harmony messages for the second turn's input
1428
req_2 = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
1432
- input_messages_2, _ = (
1433
- serving_chat.openai_serving_render._make_request_with_harmony(req_2)
1429
+ input_messages_2, _ = serving_chat.online_renderer._make_request_with_harmony(
1430
+ req_2
1431
)
1432
verify_harmony_messages(
1433
input_messages_2,
@@ -1452,9 +1449,7 @@ class TestServingChatWithHarmony:
1449
{"role": "user", "content": "Hello"},
1450
]
1451
req = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
1455
- input_messages, _ = (
1456
- serving_chat.openai_serving_render._make_request_with_harmony(req)
1457
- )
1452
+ input_messages, _ = serving_chat.online_renderer._make_request_with_harmony(req)
1453
verify_harmony_messages(
1454
input_messages,
1455
[
@@ -1478,9 +1473,7 @@ class TestServingChatWithHarmony:
1473
req = ChatCompletionRequest(
1474
model=MODEL_NAME, messages=messages, tools=weather_tools
1475
)
1481
- input_messages, _ = (
1482
- serving_chat.openai_serving_render._make_request_with_harmony(req)
1483
- )
1476
+ input_messages, _ = serving_chat.online_renderer._make_request_with_harmony(req)
1477
verify_harmony_messages(
1478
input_messages,
1479
[
@@ -1503,9 +1496,7 @@ class TestServingChatWithHarmony:
1496
1497
# Test the Harmony messages for the first turn's input
1498
req = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools)
1506
- input_messages, _ = (
1507
- serving_chat.openai_serving_render._make_request_with_harmony(req)
1508
- )
1499
+ input_messages, _ = serving_chat.online_renderer._make_request_with_harmony(req)
1500
verify_harmony_messages(
1501
input_messages,
1502
[
@@ -1549,8 +1540,8 @@ class TestServingChatWithHarmony:
1540
1541
# Test the Harmony messages for the second turn's input
1542
req_2 = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools)
1552
- input_messages_2, _ = (
1553
- serving_chat.openai_serving_render._make_request_with_harmony(req_2)
1543
+ input_messages_2, _ = serving_chat.online_renderer._make_request_with_harmony(
1544
+ req_2
1545
)
1546
verify_harmony_messages(
1547
input_messages_2,
@@ -1588,9 +1579,7 @@ class TestServingChatWithHarmony:
1579
1580
# Test the Harmony messages for the first turn's input
1581
req = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools)
1591
- input_messages, _ = (
1592
- serving_chat.openai_serving_render._make_request_with_harmony(req)
1593
- )
1582
+ input_messages, _ = serving_chat.online_renderer._make_request_with_harmony(req)
1583
verify_harmony_messages(
1584
input_messages,
1585
[
@@ -1634,8 +1623,8 @@ class TestServingChatWithHarmony:
1623
1624
# Test the Harmony messages for the second turn's input
1625
req_2 = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools)
1637
- input_messages_2, _ = (
1638
- serving_chat.openai_serving_render._make_request_with_harmony(req_2)
1626
+ input_messages_2, _ = serving_chat.online_renderer._make_request_with_harmony(
1627
+ req_2
1628
)
1629
verify_harmony_messages(
1630
input_messages_2,
@@ -1686,8 +1675,8 @@ class TestServingChatWithHarmony:
1675
1676
# Test the Harmony messages for the third turn's input
1677
req_3 = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools)
1689
- input_messages_3, _ = (
1690
- serving_chat.openai_serving_render._make_request_with_harmony(req_3)
1678
+ input_messages_3, _ = serving_chat.online_renderer._make_request_with_harmony(
1679
+ req_3
1680
)
1681
verify_harmony_messages(
1682
input_messages_3,
@@ -1751,8 +1740,8 @@ class TestServingChatWithHarmony:
1740
1741
# Test the Harmony messages for the fourth turn's input
1742
req_4 = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools)
1754
- input_messages_4, _ = (
1755
- serving_chat.openai_serving_render._make_request_with_harmony(req_4)
1743
+ input_messages_4, _ = serving_chat.online_renderer._make_request_with_harmony(
1744
+ req_4
1745
)
1746
verify_harmony_messages(
1747
input_messages_4,
@@ -1802,9 +1791,7 @@ class TestServingChatWithHarmony:
1791
},
1792
]
1793
req = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
1805
- input_messages, _ = (
1806
- serving_chat.openai_serving_render._make_request_with_harmony(req)
1807
- )
1794
+ input_messages, _ = serving_chat.online_renderer._make_request_with_harmony(req)
1795
1796
verify_harmony_messages(
1797
input_messages,
@@ -1835,9 +1822,7 @@ class TestServingChatWithHarmony:
1822
},
1823
]
1824
req = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
1838
- input_messages, _ = (
1839
- serving_chat.openai_serving_render._make_request_with_harmony(req)
1840
- )
1825
+ input_messages, _ = serving_chat.online_renderer._make_request_with_harmony(req)
1826
1827
verify_harmony_messages(
1828
input_messages,
@@ -1866,9 +1851,7 @@ class TestServingChatWithHarmony:
1851
},
1852
]
1853
req = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
1869
- input_messages, _ = (
1870
- serving_chat.openai_serving_render._make_request_with_harmony(req)
1871
- )
1854
+ input_messages, _ = serving_chat.online_renderer._make_request_with_harmony(req)
1855
1856
verify_harmony_messages(
1857
input_messages,
@@ -1898,14 +1881,14 @@ async def test_tool_choice_validation_without_parser():
1881
engine_client=mock_engine,
1882
base_model_paths=BASE_MODEL_PATHS,
1883
)
1901
- openai_serving_render = _build_serving_render(mock_engine, models.registry)
1884
+ online_renderer = _build_online_renderer(mock_engine, models.registry)
1885
1886
# Create serving_chat without tool_parser (enable_auto_tools=False)
1887
serving_chat = OpenAIServingChat(
1888
mock_engine,
1889
models,
1890
response_role="assistant",
1908
- openai_serving_render=openai_serving_render,
1891
+ online_renderer=online_renderer,
1892
chat_template=CHAT_TEMPLATE,
1893
chat_template_content_format="auto",
1894
request_logger=None,
@@ -1967,13 +1950,13 @@ async def test_streaming_n_gt1_independent_tool_parsers():
1950
engine_client=mock_engine,
1951
base_model_paths=BASE_MODEL_PATHS,
1952
)
1970
- openai_serving_render = _build_serving_render(mock_engine, models.registry)
1953
+ online_renderer = _build_online_renderer(mock_engine, models.registry)
1954
1955
serving_chat = OpenAIServingChat(
1956
mock_engine,
1957
models,
1958
response_role="assistant",
1976
- openai_serving_render=openai_serving_render,
1959
+ online_renderer=online_renderer,
1960
chat_template=CHAT_TEMPLATE,
1961
chat_template_content_format="auto",
1962
request_logger=None,
tests/entrypoints/openai/completion/test_completion_error.py
+47
-12
@@ -14,9 +14,11 @@ from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
14
from vllm.entrypoints.openai.engine.protocol import GenerationError
15
from vllm.entrypoints.openai.models.protocol import BaseModelPath
16
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
17
-from vllm.entrypoints.serve.render.serving import OpenAIServingRender
17
+from vllm.entrypoints.serve.render.serving import ServingRender
18
from vllm.outputs import CompletionOutput, RequestOutput
19
from vllm.renderers.hf import HfRenderer
20
+from vllm.renderers.online_derenderer import OnlineDerenderer
21
+from vllm.renderers.online_renderer import OnlineRenderer
22
from vllm.tokenizers.registry import cached_tokenizer_from_config
23
from vllm.v1.engine.async_llm import AsyncLLM
24
@@ -77,10 +79,9 @@ def _build_serving_completion(engine: AsyncLLM) -> OpenAIServingCompletion:
79
engine_client=engine,
80
base_model_paths=BASE_MODEL_PATHS,
81
)
80
- serving_render = OpenAIServingRender(
82
+ online_renderer = OnlineRenderer(
83
model_config=engine.model_config,
84
renderer=engine.renderer,
83
- model_registry=models.registry,
85
request_logger=None,
86
chat_template=None,
87
chat_template_content_format="auto",
@@ -88,7 +89,7 @@ def _build_serving_completion(engine: AsyncLLM) -> OpenAIServingCompletion:
89
return OpenAIServingCompletion(
90
engine,
91
models,
91
- openai_serving_render=serving_render,
92
+ online_renderer=online_renderer,
93
request_logger=None,
94
)
95
@@ -158,7 +159,7 @@ async def test_openai_completion_keeps_mm_cache_for_engine_execution():
159
mock_engine.renderer = _build_renderer(mock_engine.model_config)
160
161
serving_completion = _build_serving_completion(mock_engine)
161
- serving_completion.openai_serving_render.preprocess_completion = AsyncMock(
162
+ serving_completion.online_renderer.preprocess_completion = AsyncMock(
163
return_value=[{"prompt_token_ids": [1, 2, 3]}]
164
)
165
@@ -171,13 +172,48 @@ async def test_openai_completion_keeps_mm_cache_for_engine_execution():
172
173
assert isinstance(result, list)
174
assert (
174
- serving_completion.openai_serving_render.preprocess_completion.call_args.kwargs[
175
+ serving_completion.online_renderer.preprocess_completion.call_args.kwargs[
176
"skip_mm_cache"
177
]
178
is False
179
)
180
181
182
+def _build_serving_render(engine: AsyncLLM) -> ServingRender:
183
+ models = OpenAIServingModels(
184
+ engine_client=engine,
185
+ base_model_paths=BASE_MODEL_PATHS,
186
+ )
187
+ online_renderer = OnlineRenderer(
188
+ model_config=engine.model_config,
189
+ renderer=engine.renderer,
190
+ request_logger=None,
191
+ chat_template=None,
192
+ chat_template_content_format="auto",
193
+ )
194
+ online_derenderer = OnlineDerenderer(
195
+ model_config=engine.model_config,
196
+ renderer=engine.renderer,
197
+ request_logger=None,
198
+ chat_template=None,
199
+ chat_template_content_format="auto",
200
+ )
201
+
202
+ serving_render = ServingRender(models, online_renderer, online_derenderer)
203
+
204
+ async def _fake_preprocess_chat(*args, **kwargs):
205
+ # return conversation, engine_inputs
206
+ return (
207
+ [{"role": "user", "content": "Test"}],
208
+ [{"prompt_token_ids": [1, 2, 3]}],
209
+ )
210
+
211
+ serving_render.online_renderer.preprocess_chat = AsyncMock(
212
+ side_effect=_fake_preprocess_chat
213
+ )
214
+ return serving_render
215
+
216
+
217
@pytest.mark.asyncio
218
async def test_renderer_only_completion_request_skips_mm_cache():
219
mock_engine = MagicMock(spec=AsyncLLM)
@@ -186,8 +222,9 @@ async def test_renderer_only_completion_request_skips_mm_cache():
222
mock_engine.input_processor = MagicMock()
223
mock_engine.renderer = _build_renderer(mock_engine.model_config)
224
189
- serving_completion = _build_serving_completion(mock_engine)
190
- serving_completion.openai_serving_render.preprocess_completion = AsyncMock(
225
+ serving_render = _build_serving_render(mock_engine)
226
+
227
+ serving_render.online_renderer.preprocess_completion = AsyncMock(
228
return_value=[{"prompt_token_ids": [1, 2, 3]}]
229
)
230
@@ -196,13 +233,11 @@ async def test_renderer_only_completion_request_skips_mm_cache():
233
prompt="Test prompt",
234
)
235
199
- result = await serving_completion.openai_serving_render.render_completion_request(
200
- request
201
- )
236
+ result = await serving_render.render_completion_request(request)
237
238
assert isinstance(result, list)
239
assert (
205
- serving_completion.openai_serving_render.preprocess_completion.call_args.kwargs[
240
+ serving_render.online_renderer.preprocess_completion.call_args.kwargs[
241
"skip_mm_cache"
242
]
243
is True
tests/entrypoints/openai/completion/test_lora_resolvers.py
+3
-4
@@ -14,10 +14,10 @@ from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
14
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
15
from vllm.entrypoints.openai.models.protocol import BaseModelPath
16
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
17
-from vllm.entrypoints.serve.render.serving import OpenAIServingRender
17
from vllm.lora.request import LoRARequest
18
from vllm.lora.resolver import LoRAResolver, LoRAResolverRegistry
19
from vllm.renderers.hf import HfRenderer
20
+from vllm.renderers.online_renderer import OnlineRenderer
21
from vllm.tokenizers.registry import cached_tokenizer_from_config
22
from vllm.v1.engine.async_llm import AsyncLLM
23
@@ -144,16 +144,15 @@ def mock_serving_setup():
144
base_model_paths=BASE_MODEL_PATHS,
145
)
146
147
- serving_render = OpenAIServingRender(
147
+ online_renderer = OnlineRenderer(
148
model_config=mock_engine.model_config,
149
renderer=mock_engine.renderer,
150
- model_registry=models.registry,
150
request_logger=None,
151
chat_template=None,
152
chat_template_content_format="auto",
153
)
154
serving_completion = OpenAIServingCompletion(
156
- mock_engine, models, openai_serving_render=serving_render, request_logger=None
155
+ mock_engine, models, online_renderer=online_renderer, request_logger=None
156
)
157
158
return mock_engine, serving_completion
tests/entrypoints/openai/responses/test_serving_responses.py
+4
-4
@@ -230,7 +230,7 @@ class TestInitializeToolSessions:
230
instance = OpenAIServingResponses(
231
engine_client=engine_client,
232
models=models,
233
- openai_serving_render=MagicMock(),
233
+ online_renderer=MagicMock(),
234
request_logger=None,
235
chat_template=None,
236
chat_template_content_format="auto",
@@ -316,7 +316,7 @@ class TestValidateGeneratorInput:
316
instance = OpenAIServingResponses(
317
engine_client=engine_client,
318
models=models,
319
- openai_serving_render=MagicMock(),
319
+ online_renderer=MagicMock(),
320
request_logger=None,
321
chat_template=None,
322
chat_template_content_format="auto",
@@ -379,7 +379,7 @@ async def test_reasoning_tokens_counted_for_text_reasoning_model(monkeypatch):
379
serving = OpenAIServingResponses(
380
engine_client=engine_client,
381
models=models,
382
- openai_serving_render=MagicMock(),
382
+ online_renderer=MagicMock(),
383
request_logger=None,
384
chat_template=None,
385
chat_template_content_format="auto",
@@ -684,7 +684,7 @@ def _make_serving_instance_with_reasoning():
684
serving = OpenAIServingResponses(
685
engine_client=engine_client,
686
models=models,
687
- openai_serving_render=MagicMock(),
687
+ online_renderer=MagicMock(),
688
request_logger=None,
689
chat_template=None,
690
chat_template_content_format="auto",
tests/entrypoints/pooling/embed/test_io_processor.py
+6
-6
@@ -428,8 +428,8 @@ class TestPreProcessCohereOnline:
428
handler._get_task_instruction_prefix = lambda _input_type: None
429
handler._has_chat_template = lambda: False
430
handler._preprocess_cmpl_online = preprocess_cmpl_online
431
- handler._batch_render_chat = lambda *_args, **_kwargs: (
432
- pytest.fail("text-only request should not require chat rendering")
431
+ handler._batch_render_chat = lambda *_args, **_kwargs: pytest.fail(
432
+ "text-only request should not require chat rendering"
433
)
434
435
handler._pre_process_cohere_online(ctx)
@@ -448,8 +448,8 @@ class TestPreProcessCohereOnline:
448
449
handler._get_task_instruction_prefix = lambda _input_type: "query: "
450
handler._has_chat_template = lambda: False
451
- handler._batch_render_chat = lambda *_args, **_kwargs: (
452
- pytest.fail("chat rendering should be skipped without a template")
451
+ handler._batch_render_chat = lambda *_args, **_kwargs: pytest.fail(
452
+ "chat rendering should be skipped without a template"
453
)
454
handler._preprocess_cmpl_online = preprocess_cmpl
455
@@ -485,8 +485,8 @@ class TestPreProcessCohereOnline:
485
handler._get_task_instruction_prefix = lambda _input_type: "query: "
486
handler._has_chat_template = lambda: True
487
handler._batch_render_chat = batch_render_chat
488
- handler._preprocess_cmpl_online = lambda *_args, **_kwargs: (
489
- pytest.fail("completion path should be skipped when a template exists")
488
+ handler._preprocess_cmpl_online = lambda *_args, **_kwargs: pytest.fail(
489
+ "completion path should be skipped when a template exists"
490
)
491
492
handler._pre_process_cohere_online(ctx)
tests/entrypoints/serve/disagg/test_generate_stream.py
+5
-8
@@ -17,10 +17,10 @@ from vllm.entrypoints.serve.disagg.protocol import (
17
GenerateResponse,
18
)
19
from vllm.entrypoints.serve.disagg.serving import ServingTokens
20
-from vllm.entrypoints.serve.render.serving import OpenAIServingRender
20
from vllm.logprobs import Logprob
21
from vllm.outputs import CompletionOutput, RequestOutput
22
from vllm.renderers import renderer_from_config
23
+from vllm.renderers.online_renderer import OnlineRenderer
24
from vllm.sampling_params import SamplingParams
25
from vllm.v1.engine.async_llm import AsyncLLM
26
@@ -92,10 +92,9 @@ def _build_serving_tokens(engine: AsyncLLM, **kwargs) -> ServingTokens:
92
engine_client=engine,
93
base_model_paths=BASE_MODEL_PATHS,
94
)
95
- serving_render = OpenAIServingRender(
95
+ online_renderer = OnlineRenderer(
96
model_config=engine.model_config,
97
renderer=engine.renderer,
98
- model_registry=models.registry,
98
request_logger=None,
99
chat_template=None,
100
chat_template_content_format="auto",
@@ -103,7 +102,7 @@ def _build_serving_tokens(engine: AsyncLLM, **kwargs) -> ServingTokens:
102
serving = ServingTokens(
103
engine,
104
models,
106
- openai_serving_render=serving_render,
105
+ online_renderer=online_renderer,
106
request_logger=None,
107
**kwargs,
108
)
@@ -111,7 +110,7 @@ def _build_serving_tokens(engine: AsyncLLM, **kwargs) -> ServingTokens:
110
async def _fake_preprocess(*args, **kwargs):
111
return [{"prompt_token_ids": [1, 2, 3]}]
112
114
- serving.openai_serving_render.preprocess_completion = AsyncMock(
113
+ serving.online_renderer.preprocess_completion = AsyncMock(
114
side_effect=_fake_preprocess
115
)
116
return serving
@@ -199,9 +198,7 @@ async def test_serve_tokens_skips_mm_cache_for_remote_engine_execution():
198
199
assert isinstance(response, GenerateResponse)
200
assert (
202
- serving.openai_serving_render.preprocess_completion.call_args.kwargs[
203
- "skip_mm_cache"
204
- ]
201
+ serving.online_renderer.preprocess_completion.call_args.kwargs["skip_mm_cache"]
202
is True
203
)
204
tests/entrypoints/serve/tokenize/test_serving_tokenization.py
+7
-11
@@ -10,12 +10,12 @@ import pytest
10
from vllm.config.multimodal import MultiModalConfig
11
from vllm.entrypoints.openai.models.protocol import BaseModelPath
12
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
13
-from vllm.entrypoints.serve.render.serving import OpenAIServingRender
13
from vllm.entrypoints.serve.tokenize.protocol import (
14
TokenizeChatRequest,
15
TokenizeCompletionRequest,
16
)
17
from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
18
+from vllm.renderers.online_renderer import OnlineRenderer
19
from vllm.v1.engine.async_llm import AsyncLLM
20
21
MODEL_NAME = "openai-community/gpt2"
@@ -63,18 +63,16 @@ def _build_serving_tokenization(engine: AsyncLLM) -> ServingTokenization:
63
engine_client=engine,
64
base_model_paths=BASE_MODEL_PATHS,
65
)
66
- serving_render = OpenAIServingRender(
66
+ online_renderer = OnlineRenderer(
67
model_config=engine.model_config,
68
renderer=engine.renderer,
69
- model_registry=models.registry,
69
request_logger=None,
70
chat_template=None,
71
chat_template_content_format="auto",
72
)
73
return ServingTokenization(
74
models,
76
- openai_serving_render=serving_render,
77
- request_logger=None,
75
+ online_renderer=online_renderer,
76
chat_template=None,
77
chat_template_content_format="auto",
78
)
@@ -89,7 +87,7 @@ async def test_tokenize_chat_skips_mm_cache_for_renderer_only_path():
87
mock_engine.renderer = MagicMock()
88
89
serving = _build_serving_tokenization(mock_engine)
92
- serving.openai_serving_render.preprocess_chat = AsyncMock(
90
+ serving.online_renderer.preprocess_chat = AsyncMock(
91
return_value=(
92
[{"role": "user", "content": "Test"}],
93
[{"prompt_token_ids": [1, 2, 3]}],
@@ -105,7 +103,7 @@ async def test_tokenize_chat_skips_mm_cache_for_renderer_only_path():
103
104
assert response.tokens == [1, 2, 3]
105
assert (
108
- serving.openai_serving_render.preprocess_chat.call_args.kwargs["skip_mm_cache"]
106
+ serving.online_renderer.preprocess_chat.call_args.kwargs["skip_mm_cache"]
107
is True
108
)
109
@@ -119,7 +117,7 @@ async def test_tokenize_completion_skips_mm_cache_for_renderer_only_path():
117
mock_engine.renderer = MagicMock()
118
119
serving = _build_serving_tokenization(mock_engine)
122
- serving.openai_serving_render.preprocess_completion = AsyncMock(
120
+ serving.online_renderer.preprocess_completion = AsyncMock(
121
return_value=[{"prompt_token_ids": [1, 2, 3]}]
122
)
123
@@ -132,8 +130,6 @@ async def test_tokenize_completion_skips_mm_cache_for_renderer_only_path():
130
131
assert response.tokens == [1, 2, 3]
132
assert (
135
- serving.openai_serving_render.preprocess_completion.call_args.kwargs[
136
- "skip_mm_cache"
137
- ]
133
+ serving.online_renderer.preprocess_completion.call_args.kwargs["skip_mm_cache"]
134
is True
135
)
tests/entrypoints/speech_to_text/test_speech_to_text_cancellation.py
+2
-2
@@ -99,8 +99,8 @@ async def test_non_streaming_cancel_advances_all_chunk_generators():
99
engine_client = SimpleNamespace(
100
errored=False,
101
generate=Mock(
102
- side_effect=lambda *_args, **_kwargs: (
103
- _records_start_then_never_finishes(started_request_ids, _args[2])
102
+ side_effect=lambda *_args, **_kwargs: _records_start_then_never_finishes(
103
+ started_request_ids, _args[2]
104
)
105
),
106
abort=AsyncMock(),
tests/v1/engine/test_async_llm.py
+3
-4
@@ -512,12 +512,11 @@ async def test_header_dp_rank_argument():
512
)
513
514
# Create render serving instance (required by OpenAIServingChat)
515
- from vllm.entrypoints.serve.render.serving import OpenAIServingRender
515
+ from vllm.renderers.online_renderer import OnlineRenderer
516
517
- serving_render = OpenAIServingRender(
517
+ online_renderer = OnlineRenderer(
518
model_config=engine.model_config,
519
renderer=engine.renderer,
520
- model_registry=models.registry,
520
request_logger=None,
521
chat_template=None,
522
chat_template_content_format="auto",
@@ -528,7 +527,7 @@ async def test_header_dp_rank_argument():
527
engine_client=engine,
528
models=models,
529
response_role="assistant",
531
- openai_serving_render=serving_render,
530
+ online_renderer=online_renderer,
531
chat_template=None,
532
chat_template_content_format="auto",
533
request_logger=None,
vllm/entrypoints/anthropic/serving.py
+4
-6
@@ -10,7 +10,7 @@ import logging
10
import time
11
import uuid
12
from collections.abc import AsyncGenerator
13
-from typing import TYPE_CHECKING, Any
13
+from typing import Any
14
15
import jinja2
16
from fastapi import Request
@@ -48,9 +48,7 @@ from vllm.entrypoints.openai.engine.protocol import (
48
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
49
from vllm.entrypoints.serve.utils.api_utils import sanitize_message
50
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
51
-
52
-if TYPE_CHECKING:
53
- from vllm.entrypoints.serve.render.serving import OpenAIServingRender
51
+from vllm.renderers.online_renderer import OnlineRenderer
52
53
logger = logging.getLogger(__name__)
54
@@ -111,7 +109,7 @@ class AnthropicServingMessages(OpenAIServingChat):
109
models: OpenAIServingModels,
110
response_role: str,
111
*,
114
- openai_serving_render: "OpenAIServingRender",
112
+ online_renderer: "OnlineRenderer",
113
request_logger: RequestLogger | None,
114
chat_template: str | None,
115
chat_template_content_format: ChatTemplateContentFormatOption,
@@ -127,7 +125,7 @@ class AnthropicServingMessages(OpenAIServingChat):
125
engine_client=engine_client,
126
models=models,
127
response_role=response_role,
130
- openai_serving_render=openai_serving_render,
128
+ online_renderer=online_renderer,
129
request_logger=request_logger,
130
chat_template=chat_template,
131
chat_template_content_format=chat_template_content_format,
vllm/entrypoints/generate/api_router.py
+6
-6
@@ -87,7 +87,7 @@ async def init_generate_state(
87
tool_server = None
88
resolved_chat_template = load_chat_template(args.chat_template)
89
90
- # Render endpoints are always backed by OpenAIServingRender so that
90
+ # Render endpoints are always backed by OnlineRenderer so that
91
# /v1/chat/completions/render and /v1/completions/render work on both
92
# generate-mode and render-only servers. Created in init_app_state.
93
@@ -95,7 +95,7 @@ async def init_generate_state(
95
OpenAIServingResponses(
96
engine_client,
97
state.openai_serving_models,
98
- state.openai_serving_render,
98
+ state.online_renderer,
99
request_logger=request_logger,
100
chat_template=resolved_chat_template,
101
chat_template_content_format=args.chat_template_content_format,
@@ -116,7 +116,7 @@ async def init_generate_state(
116
engine_client=engine_client,
117
models=state.openai_serving_models,
118
response_role=args.response_role,
119
- openai_serving_render=state.openai_serving_render,
119
+ online_renderer=state.online_renderer,
120
request_logger=request_logger,
121
chat_template=resolved_chat_template,
122
chat_template_content_format=args.chat_template_content_format,
@@ -146,7 +146,7 @@ async def init_generate_state(
146
OpenAIServingCompletion(
147
engine_client,
148
state.openai_serving_models,
149
- openai_serving_render=state.openai_serving_render,
149
+ online_renderer=state.online_renderer,
150
request_logger=request_logger,
151
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
152
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
@@ -160,7 +160,7 @@ async def init_generate_state(
160
engine_client,
161
state.openai_serving_models,
162
args.response_role,
163
- openai_serving_render=state.openai_serving_render,
163
+ online_renderer=state.online_renderer,
164
request_logger=request_logger,
165
chat_template=resolved_chat_template,
166
chat_template_content_format=args.chat_template_content_format,
@@ -179,7 +179,7 @@ async def init_generate_state(
179
ServingTokens(
180
engine_client,
181
state.openai_serving_models,
182
- state.openai_serving_render,
182
+ state.online_renderer,
183
request_logger=request_logger,
184
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
185
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
vllm/entrypoints/openai/api_server.py
+50
-8
@@ -32,7 +32,7 @@ from vllm.entrypoints.openai.engine.protocol import GenerationError
32
from vllm.entrypoints.openai.models.protocol import BaseModelPath
33
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
34
from vllm.entrypoints.serve.elastic_ep.middleware import ScalingMiddleware
35
-from vllm.entrypoints.serve.render.serving import OpenAIServingRender
35
+from vllm.entrypoints.serve.render.serving import ServingRender
36
from vllm.entrypoints.serve.sagemaker.api_router import sagemaker_standards_bootstrap
37
from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
38
from vllm.entrypoints.serve.utils.api_utils import (
@@ -55,6 +55,8 @@ from vllm.entrypoints.serve.utils.server_utils import (
55
from vllm.exceptions import VLLMValidationError
56
from vllm.logger import init_logger
57
from vllm.reasoning import ReasoningParserManager
58
+from vllm.renderers.online_derenderer import OnlineDerenderer
59
+from vllm.renderers.online_renderer import OnlineRenderer
60
from vllm.tasks import POOLING_TASKS, SupportedTask
61
from vllm.tool_parsers import ToolParserManager
62
from vllm.tracing import instrument
@@ -360,10 +362,24 @@ async def init_app_state(
362
)
363
await state.openai_serving_models.init_static_loras()
364
363
- state.openai_serving_render = OpenAIServingRender(
365
+ state.online_renderer = OnlineRenderer(
366
+ model_config=engine_client.model_config,
367
+ renderer=engine_client.renderer,
368
+ request_logger=request_logger,
369
+ chat_template=resolved_chat_template,
370
+ chat_template_content_format=args.chat_template_content_format,
371
+ trust_request_chat_template=args.trust_request_chat_template,
372
+ enable_auto_tools=args.enable_auto_tool_choice,
373
+ exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
374
+ tool_parser=args.tool_call_parser,
375
+ reasoning_parser=args.structured_outputs_config.reasoning_parser,
376
+ default_chat_template_kwargs=args.default_chat_template_kwargs,
377
+ log_error_stack=args.log_error_stack,
378
+ )
379
+
380
+ state.online_derenderer = OnlineDerenderer(
381
model_config=engine_client.model_config,
382
renderer=engine_client.renderer,
366
- model_registry=state.openai_serving_models.registry,
383
request_logger=request_logger,
384
chat_template=resolved_chat_template,
385
chat_template_content_format=args.chat_template_content_format,
@@ -378,13 +394,19 @@ async def init_app_state(
394
395
state.serving_tokenization = ServingTokenization(
396
state.openai_serving_models,
381
- state.openai_serving_render,
397
+ state.online_renderer,
398
request_logger=request_logger,
399
chat_template=resolved_chat_template,
400
chat_template_content_format=args.chat_template_content_format,
401
default_chat_template_kwargs=args.default_chat_template_kwargs,
402
trust_request_chat_template=args.trust_request_chat_template,
403
)
404
+ state.serving_render = ServingRender(
405
+ state.openai_serving_models,
406
+ state.online_renderer,
407
+ state.online_derenderer,
408
+ request_logger=request_logger,
409
+ )
410
411
if "generate" in supported_tasks:
412
from vllm.entrypoints.generate.api_router import init_generate_state
@@ -423,8 +445,8 @@ async def init_render_app_state(
445
"""
446
from vllm.entrypoints.chat_utils import load_chat_template
447
from vllm.entrypoints.openai.models.serving import OpenAIModelRegistry
426
- from vllm.entrypoints.serve.render.serving import OpenAIServingRender
448
from vllm.renderers import renderer_from_config
449
+ from vllm.renderers.online_renderer import OnlineRenderer
450
451
served_model_names = args.served_model_name or [args.model]
452
model_registry = OpenAIModelRegistry(
@@ -443,10 +465,24 @@ async def init_render_app_state(
465
renderer = renderer_from_config(vllm_config)
466
resolved_chat_template = load_chat_template(args.chat_template)
467
446
- state.openai_serving_render = OpenAIServingRender(
468
+ state.online_renderer = OnlineRenderer(
469
+ model_config=vllm_config.model_config,
470
+ renderer=renderer,
471
+ request_logger=request_logger,
472
+ chat_template=resolved_chat_template,
473
+ chat_template_content_format=args.chat_template_content_format,
474
+ trust_request_chat_template=args.trust_request_chat_template,
475
+ enable_auto_tools=args.enable_auto_tool_choice,
476
+ exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
477
+ tool_parser=args.tool_call_parser,
478
+ reasoning_parser=args.reasoning_parser,
479
+ default_chat_template_kwargs=args.default_chat_template_kwargs,
480
+ log_error_stack=args.log_error_stack,
481
+ )
482
+
483
+ state.online_derenderer = OnlineDerenderer(
484
model_config=vllm_config.model_config,
485
renderer=renderer,
449
- model_registry=model_registry,
486
request_logger=request_logger,
487
chat_template=resolved_chat_template,
488
chat_template_content_format=args.chat_template_content_format,
@@ -462,13 +498,19 @@ async def init_render_app_state(
498
state.openai_serving_models = model_registry
499
state.serving_tokenization = ServingTokenization(
500
model_registry,
465
- state.openai_serving_render,
501
+ state.online_renderer,
502
request_logger=request_logger,
503
chat_template=resolved_chat_template,
504
chat_template_content_format=args.chat_template_content_format,
505
default_chat_template_kwargs=args.default_chat_template_kwargs,
506
trust_request_chat_template=args.trust_request_chat_template,
507
)
508
+ state.serving_render = ServingRender(
509
+ model_registry,
510
+ state.online_renderer,
511
+ state.online_derenderer,
512
+ request_logger=request_logger,
513
+ )
514
515
state.vllm_config = vllm_config
516
# Disable stats logging — there is no engine to poll.
vllm/entrypoints/openai/chat_completion/batch_serving.py
+12
-12
@@ -47,7 +47,7 @@ class OpenAIServingChatBatch(OpenAIServingChat):
47
"""Validate the model and preprocess a batched chat completion request.
48
49
Performs engine-aware checks then delegates per-conversation
50
- preprocessing to OpenAIServingRender, validating the chat template
50
+ preprocessing to OnlineRenderer, validating the chat template
51
once for the whole batch.
52
53
Returns:
@@ -62,19 +62,19 @@ class OpenAIServingChatBatch(OpenAIServingChat):
62
if self.engine_client.errored:
63
raise self.engine_client.dead_error
64
65
- render = self.openai_serving_render
65
+ renderer = self.online_renderer
66
67
- if not render.use_harmony:
67
+ if not renderer.use_harmony:
68
# Common case: validate the chat template once for the whole batch.
69
- error_check_ret = render.validate_chat_template(
69
+ error_check_ret = renderer.validate_chat_template(
70
request_chat_template=request.chat_template,
71
chat_template_kwargs=request.chat_template_kwargs,
72
- trust_request_chat_template=render.trust_request_chat_template,
72
+ trust_request_chat_template=renderer.trust_request_chat_template,
73
)
74
if error_check_ret is not None:
75
return error_check_ret
76
77
- parser = render.parser
77
+ parser = renderer.parser
78
tool_dicts: list[dict] | None = None
79
80
all_conversations: list[list[ConversationMessage]] = []
@@ -82,17 +82,17 @@ class OpenAIServingChatBatch(OpenAIServingChat):
82
83
for messages in request.messages:
84
single_request = request.to_chat_completion_request(messages)
85
- if render.use_harmony:
86
- conversation, engine_prompts = render._make_request_with_harmony(
85
+ if renderer.use_harmony:
86
+ conversation, engine_prompts = renderer._make_request_with_harmony(
87
single_request, should_include_tools=tool_dicts is not None
88
)
89
else:
90
- conversation, engine_prompts = await render.preprocess_chat(
90
+ conversation, engine_prompts = await renderer.preprocess_chat(
91
single_request,
92
messages,
93
- default_template=render.chat_template,
94
- default_template_content_format=render.chat_template_content_format,
95
- default_template_kwargs=render.default_chat_template_kwargs,
93
+ default_template=renderer.chat_template,
94
+ default_template_content_format=renderer.chat_template_content_format,
95
+ default_template_kwargs=renderer.default_chat_template_kwargs,
96
tool_dicts=tool_dicts,
97
parser=parser,
98
)
vllm/entrypoints/openai/chat_completion/serving.py
+6
-8
@@ -7,7 +7,7 @@ import time
7
from collections.abc import AsyncGenerator, AsyncIterator
8
from collections.abc import Sequence as GenericSequence
9
from http import HTTPStatus
10
-from typing import TYPE_CHECKING, Any, Final, cast
10
+from typing import Any, Final, cast
11
12
import numpy as np
13
import pybase64 as base64
@@ -61,14 +61,12 @@ from vllm.outputs import RequestOutput
61
from vllm.parser import ParserManager
62
from vllm.parser.abstract_parser import Parser
63
from vllm.renderers import ChatParams
64
+from vllm.renderers.online_renderer import OnlineRenderer
65
from vllm.sampling_params import BeamSearchParams, SamplingParams
66
from vllm.tokenizers import TokenizerLike
67
from vllm.utils.collection_utils import as_list
68
from vllm.utils.mistral import is_mistral_tool_parser
69
69
-if TYPE_CHECKING:
70
- from vllm.entrypoints.serve.render.serving import OpenAIServingRender
71
-
70
logger = init_logger(__name__)
71
72
@@ -112,7 +110,7 @@ class OpenAIServingChat(OpenAIServing):
110
models: OpenAIServingModels,
111
response_role: str,
112
*,
115
- openai_serving_render: "OpenAIServingRender",
113
+ online_renderer: "OnlineRenderer",
114
request_logger: RequestLogger | None,
115
chat_template: str | None,
116
chat_template_content_format: ChatTemplateContentFormatOption,
@@ -135,7 +133,7 @@ class OpenAIServingChat(OpenAIServing):
133
return_tokens_as_token_ids=return_tokens_as_token_ids,
134
)
135
138
- self.openai_serving_render = openai_serving_render
136
+ self.online_renderer = online_renderer
137
self.response_role = response_role
138
self.chat_template = chat_template
139
self.chat_template_content_format: Final = chat_template_content_format
@@ -212,7 +210,7 @@ class OpenAIServingChat(OpenAIServing):
210
"""
211
Validate the model and preprocess a chat completion request.
212
215
- Delegates preprocessing logic to OpenAIServingRender, adding the
213
+ Delegates preprocessing logic to OnlineRenderer, adding the
214
engine-aware checks (LoRA model validation, engine health).
215
216
Returns:
@@ -230,7 +228,7 @@ class OpenAIServingChat(OpenAIServing):
228
if self.engine_client.errored:
229
raise self.engine_client.dead_error
230
233
- return await self.openai_serving_render.render_chat(request)
231
+ return await self.online_renderer.render_chat(request)
232
233
async def create_chat_completion(
234
self,
vllm/entrypoints/openai/completion/serving.py
+6
-8
@@ -6,7 +6,7 @@ import io
6
import time
7
from collections.abc import AsyncGenerator, AsyncIterator
8
from collections.abc import Sequence as GenericSequence
9
-from typing import TYPE_CHECKING, cast
9
+from typing import cast
10
11
import numpy as np
12
import pybase64 as base64
@@ -41,14 +41,12 @@ from vllm.inputs import EngineInput
41
from vllm.logger import init_logger
42
from vllm.logprobs import Logprob
43
from vllm.outputs import RequestOutput
44
+from vllm.renderers.online_renderer import OnlineRenderer
45
from vllm.sampling_params import BeamSearchParams, SamplingParams
46
from vllm.tokenizers import TokenizerLike
47
from vllm.utils.async_utils import merge_async_iterators
48
from vllm.utils.collection_utils import as_list
49
49
-if TYPE_CHECKING:
50
- from vllm.entrypoints.serve.render.serving import OpenAIServingRender
51
-
50
logger = init_logger(__name__)
51
52
@@ -58,7 +56,7 @@ class OpenAIServingCompletion(OpenAIServing):
56
engine_client: EngineClient,
57
models: OpenAIServingModels,
58
*,
61
- openai_serving_render: "OpenAIServingRender",
59
+ online_renderer: "OnlineRenderer",
60
request_logger: RequestLogger | None,
61
return_tokens_as_token_ids: bool = False,
62
enable_prompt_tokens_details: bool = False,
@@ -71,7 +69,7 @@ class OpenAIServingCompletion(OpenAIServing):
69
return_tokens_as_token_ids=return_tokens_as_token_ids,
70
)
71
74
- self.openai_serving_render = openai_serving_render
72
+ self.online_renderer = online_renderer
73
self.enable_prompt_tokens_details = enable_prompt_tokens_details
74
self.enable_force_include_usage = enable_force_include_usage
75
@@ -90,7 +88,7 @@ class OpenAIServingCompletion(OpenAIServing):
88
"""
89
Validate the model and preprocess a completion request.
90
93
- Delegates preprocessing logic to OpenAIServingRender, adding the
91
+ Delegates preprocessing logic to OnlineRenderer, adding the
92
engine-aware checks (LoRA model validation, engine health).
93
94
Returns:
@@ -106,7 +104,7 @@ class OpenAIServingCompletion(OpenAIServing):
104
if self.engine_client.errored:
105
raise self.engine_client.dead_error
106
109
- return await self.openai_serving_render.render_completion(request)
107
+ return await self.online_renderer.render_completion(request)
108
109
async def create_completion(
110
self,
vllm/entrypoints/openai/engine/serving.py
+2
-6
@@ -13,12 +13,8 @@ from starlette.datastructures import Headers
13
14
from vllm.engine.protocol import EngineClient
15
from vllm.entrypoints.generate.beam_search.online import BeamSearchOnlineMixin
16
-from vllm.entrypoints.openai.chat_completion.protocol import (
17
- ChatCompletionRequest,
18
-)
19
-from vllm.entrypoints.openai.completion.protocol import (
20
- CompletionRequest,
21
-)
16
+from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
17
+from vllm.entrypoints.openai.completion.protocol import CompletionRequest
18
from vllm.entrypoints.openai.engine.protocol import (
19
ErrorResponse,
20
GenerationError,
vllm/entrypoints/openai/responses/serving.py
+5
-5
@@ -92,7 +92,6 @@ from vllm.entrypoints.openai.responses.utils import (
92
extract_function_tool_names,
93
extract_tool_types,
94
)
95
-from vllm.entrypoints.serve.render.serving import OpenAIServingRender
95
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens
96
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
97
from vllm.exceptions import VLLMValidationError
@@ -103,6 +102,7 @@ from vllm.logprobs import SampleLogprobs
102
from vllm.lora.request import LoRARequest
103
from vllm.outputs import CompletionOutput
104
from vllm.parser import Parser, ParserManager
105
+from vllm.renderers.online_renderer import OnlineRenderer
106
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
107
from vllm.tokenizers import TokenizerLike
108
from vllm.utils import random_uuid
@@ -155,7 +155,7 @@ class OpenAIServingResponses(OpenAIServing):
155
self,
156
engine_client: EngineClient,
157
models: OpenAIServingModels,
158
- openai_serving_render: OpenAIServingRender,
158
+ online_renderer: OnlineRenderer,
159
*,
160
request_logger: RequestLogger | None,
161
chat_template: str | None,
@@ -177,7 +177,7 @@ class OpenAIServingResponses(OpenAIServing):
177
return_tokens_as_token_ids=return_tokens_as_token_ids,
178
)
179
180
- self.openai_serving_render = openai_serving_render
180
+ self.online_renderer = online_renderer
181
self.chat_template = chat_template
182
self.chat_template_content_format: Final = chat_template_content_format
183
self.chat_template_kwargs = default_chat_template_kwargs or {}
@@ -633,7 +633,7 @@ class OpenAIServingResponses(OpenAIServing):
633
prev_response_output=prev_response.output if prev_response else None,
634
)
635
chat_template_kwargs = self._effective_chat_template_kwargs(request)
636
- _, engine_inputs = await self.openai_serving_render.preprocess_chat(
636
+ _, engine_inputs = await self.online_renderer.preprocess_chat(
637
request,
638
messages,
639
default_template=self.chat_template,
@@ -657,7 +657,7 @@ class OpenAIServingResponses(OpenAIServing):
657
request_input=messages,
658
)
659
chat_template_kwargs = self._effective_chat_template_kwargs(request)
660
- _, engine_inputs = await self.openai_serving_render.preprocess_chat(
660
+ _, engine_inputs = await self.online_renderer.preprocess_chat(
661
request,
662
new_messages,
663
default_template=chat_template,
vllm/entrypoints/serve/disagg/api_router.py
+1
-3
@@ -10,9 +10,7 @@ from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request, Respons
10
from fastapi.responses import JSONResponse, StreamingResponse
11
12
from vllm.engine.protocol import EngineClient
13
-from vllm.entrypoints.openai.engine.protocol import (
14
- ErrorResponse,
15
-)
13
+from vllm.entrypoints.openai.engine.protocol import ErrorResponse
14
from vllm.entrypoints.serve.disagg.protocol import (
15
GenerateRequest,
16
GenerateResponse,
vllm/entrypoints/serve/disagg/serving.py
+4
-4
@@ -36,7 +36,6 @@ from vllm.entrypoints.serve.disagg.protocol import (
36
GenerateResponseStreamChoice,
37
GenerateStreamResponse,
38
)
39
-from vllm.entrypoints.serve.render.serving import OpenAIServingRender
39
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage
40
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
41
from vllm.inputs import EngineInput, mm_input
@@ -48,6 +47,7 @@ from vllm.multimodal.inputs import (
47
PlaceholderRange,
48
)
49
from vllm.outputs import RequestOutput
50
+from vllm.renderers.online_renderer import OnlineRenderer
51
from vllm.sampling_params import RequestOutputKind, SamplingParams
52
from vllm.utils.collection_utils import as_list
53
@@ -61,7 +61,7 @@ class ServingTokens(OpenAIServing):
61
self,
62
engine_client: EngineClient,
63
models: OpenAIServingModels,
64
- openai_serving_render: OpenAIServingRender,
64
+ online_renderer: OnlineRenderer,
65
*,
66
request_logger: RequestLogger | None,
67
force_no_detokenize: bool = False,
@@ -75,7 +75,7 @@ class ServingTokens(OpenAIServing):
75
request_logger=request_logger,
76
return_tokens_as_token_ids=return_tokens_as_token_ids,
77
)
78
- self.openai_serving_render = openai_serving_render
78
+ self.online_renderer = online_renderer
79
self.enable_prompt_tokens_details = enable_prompt_tokens_details
80
self.enable_log_outputs = enable_log_outputs
81
self.force_no_detokenize = force_no_detokenize
@@ -168,7 +168,7 @@ class ServingTokens(OpenAIServing):
168
cache_salt=request.cache_salt,
169
)
170
else:
171
- (engine_input,) = await self.openai_serving_render.preprocess_completion(
171
+ (engine_input,) = await self.online_renderer.preprocess_completion(
172
request,
173
prompt_input=request.token_ids,
174
prompt_embeds=None,
vllm/entrypoints/serve/engine/typing.py
+14
-3
@@ -15,7 +15,12 @@ from vllm.entrypoints.openai.completion.protocol import (
15
CompletionResponse,
16
)
17
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
18
-from vllm.entrypoints.serve.disagg.protocol import GenerateRequest, GenerateResponse
18
+from vllm.entrypoints.serve.disagg.protocol import (
19
+ DerenderChatRequest,
20
+ DerenderCompletionRequest,
21
+ GenerateRequest,
22
+ GenerateResponse,
23
+)
24
from vllm.entrypoints.serve.tokenize.protocol import (
25
DetokenizeRequest,
26
TokenizeChatRequest,
@@ -45,11 +50,17 @@ class RendererChatRequest(RendererRequest, Protocol):
50
51
52
CompletionLikeRequest: TypeAlias = (
48
- CompletionRequest | TokenizeCompletionRequest | DetokenizeRequest
53
+ CompletionRequest
54
+ | TokenizeCompletionRequest
55
+ | DetokenizeRequest
56
+ | DerenderCompletionRequest
57
)
58
59
ChatLikeRequest: TypeAlias = (
52
- ChatCompletionRequest | BatchChatCompletionRequest | TokenizeChatRequest
60
+ ChatCompletionRequest
61
+ | BatchChatCompletionRequest
62
+ | TokenizeChatRequest
63
+ | DerenderChatRequest
64
)
65
66
SpeechToTextRequest: TypeAlias = TranscriptionRequest | TranslationRequest
vllm/entrypoints/serve/render/api_router.py
+3
-3
@@ -19,7 +19,7 @@ from vllm.entrypoints.serve.disagg.protocol import (
19
DerenderCompletionRequest,
20
GenerateRequest,
21
)
22
-from vllm.entrypoints.serve.render.serving import OpenAIServingRender
22
+from vllm.entrypoints.serve.render.serving import ServingRender
23
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
24
from vllm.logger import init_logger
25
@@ -28,8 +28,8 @@ logger = init_logger(__name__)
28
router = APIRouter()
29
30
31
-def render(request: Request) -> OpenAIServingRender | None:
32
- return getattr(request.app.state, "openai_serving_render", None)
31
+def render(request: Request) -> ServingRender | None:
32
+ return getattr(request.app.state, "serving_render", None)
33
34
35
@router.post(
vllm/entrypoints/serve/render/serving.py
+69
-687
@@ -1,44 +1,24 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import time
4
-from collections.abc import Sequence
5
-from http import HTTPStatus
6
-from typing import Any, cast
4
+from typing import cast
5
8
-from openai_harmony import Message as OpenAIMessage
9
-
10
-from vllm.config import ModelConfig
11
-from vllm.entrypoints.chat_utils import (
12
- ChatTemplateContentFormatOption,
13
- ConversationMessage,
14
-)
6
from vllm.entrypoints.openai.chat_completion.protocol import (
16
- ChatCompletionLogProbs,
7
ChatCompletionRequest,
8
ChatCompletionResponse,
19
- ChatCompletionResponseChoice,
20
- ChatMessage,
9
)
10
from vllm.entrypoints.openai.completion.protocol import (
23
- CompletionLogProbs,
11
CompletionRequest,
12
CompletionResponse,
26
- CompletionResponseChoice,
13
)
14
from vllm.entrypoints.openai.engine.protocol import (
15
ErrorResponse,
30
- ToolCall,
16
UsageInfo,
17
)
33
-from vllm.entrypoints.openai.engine.serving import resolve_token_id_placeholder
34
-from vllm.entrypoints.openai.models.serving import OpenAIModelRegistry
35
-from vllm.entrypoints.openai.parser.harmony_utils import (
36
- build_harmony_preamble,
37
- extract_instructions_from_messages,
38
- parse_chat_inputs_to_harmony_messages,
39
- render_for_completion,
18
+from vllm.entrypoints.openai.models.serving import (
19
+ OpenAIModelRegistry,
20
+ OpenAIServingModels,
21
)
41
-from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
22
from vllm.entrypoints.serve.disagg.mm_serde import encode_mm_kwargs_item
23
from vllm.entrypoints.serve.disagg.protocol import (
24
DerenderChatRequest,
@@ -47,207 +27,49 @@ from vllm.entrypoints.serve.disagg.protocol import (
27
MultiModalFeatures,
28
PlaceholderRangeInfo,
29
)
30
+from vllm.entrypoints.serve.engine.serving import BaseServing
31
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens
51
-from vllm.entrypoints.serve.utils.error_response import create_error_response
32
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
33
from vllm.inputs import (
34
EngineInput,
35
MultiModalHashes,
36
MultiModalInput,
37
MultiModalPlaceholders,
58
- PromptType,
59
- SingletonPrompt,
60
- tokens_input,
38
)
39
from vllm.logger import init_logger
63
-from vllm.parser import Parser, ParserManager
64
-from vllm.renderers import BaseRenderer, merge_kwargs
40
from vllm.renderers.inputs.preprocess import (
41
extract_prompt_components,
42
extract_prompt_len,
68
- parse_model_prompt,
69
- prompt_to_seq,
43
)
71
-from vllm.tokenizers import TokenizerLike
44
+from vllm.renderers.online_derenderer import OnlineDerenderer
45
+from vllm.renderers.online_renderer import OnlineRenderer
46
from vllm.utils import random_uuid
73
-from vllm.utils.mistral import is_mistral_tokenizer, is_mistral_tool_parser
74
-from vllm.utils.mistral import mt as _mt
47
48
logger = init_logger(__name__)
49
50
79
-def _parse_token_id_placeholder(token: str) -> int | None:
80
- """Extract token ID from a 'token_id:N' placeholder string."""
81
- if not token.startswith("token_id:"):
82
- return None
83
- try:
84
- return int(token[len("token_id:") :])
85
- except ValueError:
86
- return None
87
-
88
-
89
-def _correct_decoded_token(
90
- token_id: int, context_token_ids: list[int], tokenizer: TokenizerLike
91
-) -> str:
92
- """Use preceding tokens as context to fix U+FFFD from byte-fallback.
93
-
94
- Mirrors LogprobsProcessor._correct_decoded_token in v1/engine/logprobs.py.
95
- """
96
- max_ctx = min(len(context_token_ids), 4)
97
-
98
- for num_ctx in range(1, max_ctx + 1):
99
- context = context_token_ids[-num_ctx:]
100
- full_decoded = tokenizer.decode(context + [token_id])
101
-
102
- if full_decoded.endswith("�"):
103
- continue
104
-
105
- clean_end = len(context)
106
- for j in range(len(context) - 1, -1, -1):
107
- if tokenizer.decode([context[j]]).endswith("�"):
108
- clean_end = j
109
- else:
110
- break
111
-
112
- clean_prefix = tokenizer.decode(context[:clean_end]) if clean_end > 0 else ""
113
-
114
- if full_decoded.startswith(clean_prefix):
115
- return full_decoded[len(clean_prefix) :]
116
-
117
- common_len = 0
118
- for a, b in zip(clean_prefix, full_decoded):
119
- if a != b:
120
- break
121
- common_len += 1
122
- return full_decoded[common_len:]
123
-
124
- return ""
125
-
126
-
127
-def _resolve_logprobs(
128
- logprobs: ChatCompletionLogProbs, tokenizer: TokenizerLike
129
-) -> ChatCompletionLogProbs:
130
- """Resolve token_id:N placeholders in a ChatCompletionLogProbs object."""
131
- if logprobs.content is None:
132
- return logprobs
133
-
134
- context_token_ids: list[int] = []
135
- resolved_content = []
136
-
137
- for entry in logprobs.content:
138
- token_str, token_bytes = resolve_token_id_placeholder(entry.token, tokenizer)
139
- sampled_id = _parse_token_id_placeholder(entry.token)
140
-
141
- if token_str.endswith("�") and sampled_id is not None:
142
- token_str = _correct_decoded_token(sampled_id, context_token_ids, tokenizer)
143
- token_bytes = list(token_str.encode("utf-8"))
144
-
145
- resolved_top = []
146
- for top in entry.top_logprobs:
147
- top_str, top_bytes = resolve_token_id_placeholder(top.token, tokenizer)
148
- top_id = _parse_token_id_placeholder(top.token)
149
- if top_str.endswith("�") and top_id is not None:
150
- top_str = _correct_decoded_token(top_id, context_token_ids, tokenizer)
151
- top_bytes = list(top_str.encode("utf-8"))
152
- resolved_top.append(
153
- top.model_copy(update={"token": top_str, "bytes": top_bytes})
154
- )
155
-
156
- resolved_content.append(
157
- entry.model_copy(
158
- update={
159
- "token": token_str,
160
- "bytes": token_bytes,
161
- "top_logprobs": resolved_top,
162
- }
163
- )
164
- )
165
-
166
- if sampled_id is not None:
167
- context_token_ids.append(sampled_id)
168
-
169
- return ChatCompletionLogProbs(content=resolved_content)
170
-
171
-
172
-def _convert_chat_logprobs_to_completion_logprobs(
173
- logprobs: ChatCompletionLogProbs,
174
-) -> CompletionLogProbs:
175
- """Convert ChatCompletionLogProbs (per-token objects) to CompletionLogProbs
176
- (parallel flat lists) as required by the /v1/completions response schema."""
177
- if logprobs.content is None:
178
- return CompletionLogProbs()
179
-
180
- tokens: list[str] = []
181
- token_logprobs: list[float | None] = []
182
- top_logprobs_list: list[dict[str, float] | None] = []
183
- text_offset: list[int] = []
184
-
185
- offset = 0
186
- for entry in logprobs.content:
187
- text_offset.append(offset)
188
- tokens.append(entry.token)
189
- token_logprobs.append(entry.logprob)
190
- top_logprobs_list.append(
191
- {t.token: t.logprob for t in entry.top_logprobs}
192
- if entry.top_logprobs
193
- else None
194
- )
195
- offset += len(entry.token)
196
-
197
- return CompletionLogProbs(
198
- text_offset=text_offset,
199
- token_logprobs=token_logprobs,
200
- tokens=tokens,
201
- top_logprobs=top_logprobs_list,
202
- )
203
-
204
-
205
-class OpenAIServingRender:
51
+class ServingRender(BaseServing):
52
def __init__(
53
self,
208
- model_config: ModelConfig,
209
- renderer: BaseRenderer,
210
- model_registry: OpenAIModelRegistry,
54
+ models: OpenAIServingModels | OpenAIModelRegistry,
55
+ online_renderer: "OnlineRenderer",
56
+ online_derenderer: "OnlineDerenderer",
57
*,
212
- request_logger: RequestLogger | None,
213
- chat_template: str | None,
214
- chat_template_content_format: ChatTemplateContentFormatOption,
215
- trust_request_chat_template: bool = False,
216
- enable_auto_tools: bool = False,
217
- exclude_tools_when_tool_choice_none: bool = False,
218
- tool_parser: str | None = None,
219
- reasoning_parser: str | None = None,
220
- default_chat_template_kwargs: dict[str, Any] | None = None,
221
- log_error_stack: bool = False,
58
+ request_logger: RequestLogger | None = None,
59
) -> None:
223
- self.model_config = model_config
224
- self.renderer = renderer
225
- self.model_registry = model_registry
226
- self.request_logger = request_logger
227
- self.chat_template = chat_template
228
- self.chat_template_content_format: ChatTemplateContentFormatOption = (
229
- chat_template_content_format
230
- )
231
- self.trust_request_chat_template = trust_request_chat_template
232
- self.enable_auto_tools = enable_auto_tools
233
- self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none
234
- self.use_harmony = model_config.hf_config.model_type == "gpt_oss"
235
- self.parser: type[Parser] | None = ParserManager.get_parser(
236
- tool_parser_name=tool_parser,
237
- reasoning_parser_name=reasoning_parser,
238
- enable_auto_tools=enable_auto_tools,
239
- model_name=model_config.model,
240
- is_harmony=self.use_harmony,
241
- )
242
- self.default_chat_template_kwargs: dict[str, Any] = (
243
- default_chat_template_kwargs or {}
60
+ super().__init__(
61
+ models=models,
62
+ model_config=online_renderer.model_config,
63
+ request_logger=request_logger,
64
)
245
- self.log_error_stack = log_error_stack
246
- self.supports_browsing = False
247
- self.supports_code_interpreter = False
65
249
- self.default_sampling_params = model_config.get_diff_sampling_param()
250
- mc = model_config
66
+ self.online_renderer = online_renderer
67
+ self.online_derenderer = online_derenderer
68
+
69
+ self.default_sampling_params = (
70
+ online_renderer.model_config.get_diff_sampling_param()
71
+ )
72
+ mc = online_renderer.model_config
73
self.override_max_tokens = (
74
self.default_sampling_params.get("max_tokens")
75
if mc.generation_config not in ("auto", "vllm")
@@ -273,7 +95,7 @@ class OpenAIServingRender:
95
"Beam search is not supported by the render endpoint"
96
)
97
276
- result = await self.render_chat(request, skip_mm_cache=True)
98
+ result = await self.online_renderer.render_chat(request, skip_mm_cache=True)
99
if isinstance(result, ErrorResponse):
100
return result
101
@@ -319,90 +141,6 @@ class OpenAIServingRender:
141
priority=request.priority,
142
)
143
322
- async def render_chat(
323
- self,
324
- request: ChatCompletionRequest,
325
- *,
326
- skip_mm_cache: bool = False,
327
- ) -> tuple[list[ConversationMessage], list[EngineInput]] | ErrorResponse:
328
- """Core preprocessing logic for chat requests (no model/engine check).
329
-
330
- Called directly by render_chat_request and delegated to by
331
- OpenAIServingChat.render_chat_request after its engine-aware checks.
332
- """
333
- tokenizer = self.renderer.tokenizer
334
-
335
- tool_parser = self.parser.tool_parser_cls if self.parser is not None else None
336
-
337
- if is_mistral_tokenizer(tokenizer):
338
- # because of issues with pydantic we need to potentially
339
- # re-serialize the tool_calls field of the request
340
- _mt.maybe_serialize_tool_calls(request) # type: ignore[arg-type]
341
- _mt.truncate_tool_call_ids(request) # type: ignore[arg-type]
342
- _mt.validate_request_params(request)
343
-
344
- # Check if tool parsing is unavailable (common condition)
345
- tool_parsing_unavailable = (
346
- tool_parser is None
347
- and not is_mistral_tokenizer(tokenizer)
348
- and not self.use_harmony
349
- )
350
-
351
- # Validate tool_choice when tool parsing is required but unavailable
352
- if tool_parsing_unavailable and request.tool_choice not in (
353
- None,
354
- "none",
355
- ):
356
- if request.tool_choice == "auto" and not self.enable_auto_tools:
357
- # for hf tokenizers, "auto" tools requires
358
- # --enable-auto-tool-choice and --tool-call-parser
359
- return self.create_error_response(
360
- '"auto" tool choice requires '
361
- "--enable-auto-tool-choice and --tool-call-parser to be set"
362
- )
363
- elif request.tool_choice != "auto":
364
- # "required" or named tool requires tool parser
365
- return self.create_error_response(
366
- f'tool_choice="{request.tool_choice}" requires '
367
- "--tool-call-parser to be set"
368
- )
369
-
370
- if request.tools is None or (
371
- request.tool_choice == "none" and self.exclude_tools_when_tool_choice_none
372
- ):
373
- tool_dicts = None
374
- else:
375
- tool_dicts = [tool.model_dump() for tool in request.tools]
376
-
377
- if not self.use_harmony:
378
- # Common case.
379
- error_check_ret = self.validate_chat_template(
380
- request_chat_template=request.chat_template,
381
- chat_template_kwargs=request.chat_template_kwargs,
382
- trust_request_chat_template=self.trust_request_chat_template,
383
- )
384
- if error_check_ret is not None:
385
- return error_check_ret
386
-
387
- conversation, engine_inputs = await self.preprocess_chat(
388
- request,
389
- request.messages,
390
- default_template=self.chat_template,
391
- default_template_content_format=self.chat_template_content_format,
392
- default_template_kwargs=self.default_chat_template_kwargs,
393
- tool_dicts=tool_dicts,
394
- parser=self.parser,
395
- skip_mm_cache=skip_mm_cache,
396
- )
397
- else:
398
- # For GPT-OSS.
399
- should_include_tools = tool_dicts is not None
400
- conversation, engine_inputs = self._make_request_with_harmony(
401
- request, should_include_tools
402
- )
403
-
404
- return conversation, engine_inputs
405
-
144
async def render_completion_request(
145
self,
146
request: CompletionRequest,
@@ -415,7 +153,9 @@ class OpenAIServingRender:
153
error_check_ret = await self._check_model(request)
154
if error_check_ret is not None:
155
return error_check_ret
418
- result = await self.render_completion(request, skip_mm_cache=True)
156
+ result = await self.online_renderer.render_completion(
157
+ request, skip_mm_cache=True
158
+ )
159
if isinstance(result, ErrorResponse):
160
return result
161
generate_requests: list[GenerateRequest] = []
@@ -459,119 +199,6 @@ class OpenAIServingRender:
199
200
return generate_requests
201
462
- async def render_completion(
463
- self,
464
- request: CompletionRequest,
465
- *,
466
- skip_mm_cache: bool = False,
467
- ) -> list[EngineInput] | ErrorResponse:
468
- """Core preprocessing logic for completion requests (no model/engine check).
469
-
470
- Called directly by render_completion_request and delegated to by
471
- OpenAIServingCompletion.render_completion_request after its engine-aware checks.
472
- """
473
- # Return error for unsupported features.
474
- if request.suffix is not None:
475
- return self.create_error_response("suffix is not currently supported")
476
-
477
- if request.echo and request.prompt_embeds is not None:
478
- return self.create_error_response("Echo is unsupported with prompt embeds.")
479
-
480
- if request.prompt_logprobs is not None and request.prompt_embeds is not None:
481
- return self.create_error_response(
482
- "prompt_logprobs is not compatible with prompt embeds."
483
- )
484
-
485
- engine_inputs = await self.preprocess_completion(
486
- request,
487
- prompt_input=request.prompt,
488
- prompt_embeds=request.prompt_embeds,
489
- skip_mm_cache=skip_mm_cache,
490
- )
491
-
492
- return engine_inputs
493
-
494
- @staticmethod
495
- def _extract_mm_features(
496
- engine_input: EngineInput,
497
- ) -> MultiModalFeatures | None:
498
- """Extract multimodal metadata from a rendered engine prompt.
499
-
500
- Returns ``None`` for text-only prompts.
501
- """
502
- if engine_input.get("type") != "multimodal":
503
- return None
504
-
505
- # At this point engine_input is a MultiModalInput TypedDict.
506
- mm_engine_input = cast(MultiModalInput, engine_input)
507
- mm_hashes: MultiModalHashes = mm_engine_input["mm_hashes"]
508
- raw_placeholders: MultiModalPlaceholders = mm_engine_input["mm_placeholders"]
509
-
510
- mm_placeholders = {
511
- modality: [
512
- PlaceholderRangeInfo(offset=p.offset, length=p.length) for p in ranges
513
- ]
514
- for modality, ranges in raw_placeholders.items()
515
- }
516
-
517
- # Serialize tensor data per modality.
518
- kwargs_data: dict[str, list[str | None]] | None = None
519
- if raw_mm_kwargs := mm_engine_input.get("mm_kwargs"):
520
- kwargs_data = {}
521
- for modality, items in raw_mm_kwargs.items():
522
- kwargs_data[modality] = [
523
- encode_mm_kwargs_item(item) if item is not None else None
524
- for item in items
525
- ]
526
-
527
- return MultiModalFeatures(
528
- mm_hashes=mm_hashes,
529
- mm_placeholders=mm_placeholders,
530
- kwargs_data=kwargs_data,
531
- )
532
-
533
- def _make_request_with_harmony(
534
- self,
535
- request: ChatCompletionRequest,
536
- should_include_tools: bool = True,
537
- ):
538
- """Build Harmony (GPT-OSS) messages and engine prompt from a chat request."""
539
- messages: list[OpenAIMessage] = []
540
-
541
- # because of issues with pydantic we need to potentially
542
- # re-serialize the tool_calls field of the request
543
- # for more info: see comment in `maybe_serialize_tool_calls`
544
- _mt.maybe_serialize_tool_calls(request) # type: ignore[arg-type]
545
-
546
- chat_messages = list(request.messages)
547
- instructions, chat_messages = extract_instructions_from_messages(chat_messages)
548
-
549
- # Add system message.
550
- # NOTE: In Chat Completion API, browsing is enabled by default
551
- # if the model supports it. TODO: Support browsing.
552
- assert not self.supports_browsing
553
- assert not self.supports_code_interpreter
554
- if (reasoning_effort := request.reasoning_effort) == "none":
555
- raise ValueError(f"Harmony does not support {reasoning_effort=}")
556
- tools = request.tools if should_include_tools else None
557
- messages.extend(
558
- build_harmony_preamble(
559
- instructions=instructions,
560
- tools=tools, # type: ignore[arg-type]
561
- reasoning_effort=reasoning_effort,
562
- with_custom_tools=should_include_tools,
563
- )
564
- )
565
-
566
- # Add remaining conversation messages.
567
- messages.extend(parse_chat_inputs_to_harmony_messages(chat_messages))
568
-
569
- # Render prompt token ids.
570
- prompt_token_ids = render_for_completion(messages)
571
- engine_input = tokens_input(prompt_token_ids, cache_salt=request.cache_salt)
572
-
573
- return messages, [engine_input]
574
-
202
async def derender_chat_response(
203
self,
204
request: DerenderChatRequest,
@@ -589,97 +216,17 @@ class OpenAIServingRender:
216
if error_check_ret is not None:
217
return error_check_ret
218
592
- tokenizer = self.renderer.get_tokenizer()
593
- gen = request.generate_response
594
- chat_request = request.chat_request
595
- choices: list[ChatCompletionResponseChoice] = []
596
-
219
try:
598
- for choice in gen.choices:
599
- if not choice.token_ids:
600
- raise ValueError(
601
- f"choice {choice.index} has empty or null token_ids"
602
- )
603
-
604
- resolved_logprobs = (
605
- _resolve_logprobs(choice.logprobs, tokenizer)
606
- if choice.logprobs is not None
607
- else None
608
- )
609
-
610
- if self.parser is not None and chat_request is not None:
611
- # Parser path: decode with special tokens preserved
612
- # so the parser can see markers like </think>,
613
- # <tool_call>, or Harmony channel tokens.
614
- decoded_text = tokenizer.decode(
615
- choice.token_ids, skip_special_tokens=False
616
- )
617
-
618
- chat_template_kwargs: dict[str, Any] = {}
619
- if not self.use_harmony:
620
- chat_template_kwargs = (
621
- chat_request.build_chat_params(
622
- self.chat_template,
623
- self.chat_template_content_format,
624
- )
625
- .with_defaults(self.default_chat_template_kwargs)
626
- .chat_template_kwargs
627
- )
628
-
629
- parser = self.parser(
630
- tokenizer,
631
- chat_request.tools,
632
- chat_template_kwargs=chat_template_kwargs,
633
- )
634
- reasoning, content, tool_calls = parser.parse(
635
- decoded_text,
636
- chat_request,
637
- enable_auto_tools=self.enable_auto_tools,
638
- model_output_token_ids=choice.token_ids,
639
- )
640
-
641
- if not getattr(chat_request, "include_reasoning", True):
642
- reasoning = None
643
-
644
- tc_items = (
645
- [
646
- ToolCall(
647
- id=random_uuid(),
648
- function=tc,
649
- )
650
- for tc in tool_calls
651
- ]
652
- if tool_calls
653
- else []
654
- )
655
-
656
- message = ChatMessage(
657
- role="assistant",
658
- reasoning=reasoning,
659
- content=content,
660
- tool_calls=tc_items,
661
- )
662
- else:
663
- # No parser: plain detokenization.
664
- decoded_text = tokenizer.decode(
665
- choice.token_ids, skip_special_tokens=True
666
- )
667
- message = ChatMessage(role="assistant", content=decoded_text)
668
-
669
- choices.append(
670
- ChatCompletionResponseChoice(
671
- index=choice.index,
672
- message=message,
673
- logprobs=resolved_logprobs,
674
- finish_reason=choice.finish_reason,
675
- )
676
- )
220
+ choices = await self.online_derenderer.derender_chat(
221
+ request.generate_response, request.chat_request
222
+ )
223
except ValueError as exc:
224
return self.create_error_response(str(exc))
225
226
prompt_tokens = (
227
request.prompt_tokens if request.prompt_tokens is not None else 0
228
)
229
+ gen = request.generate_response
230
completion_tokens = sum(len(ch.token_ids) for ch in gen.choices if ch.token_ids)
231
usage = UsageInfo(
232
prompt_tokens=prompt_tokens,
@@ -718,45 +265,14 @@ class OpenAIServingRender:
265
if error_check_ret is not None:
266
return error_check_ret
267
721
- n = len(request.generate_responses)
722
- prompt_tokens_list: list[int] = (
723
- request.prompt_tokens if request.prompt_tokens is not None else [0] * n
268
+ (
269
+ choices,
270
+ total_prompt_tokens,
271
+ total_completion_tokens,
272
+ ) = await self.online_derenderer.derender_completion(
273
+ request.generate_responses, request.prompt_tokens
274
)
275
726
- tokenizer = self.renderer.get_tokenizer()
727
- choices: list[CompletionResponseChoice] = []
728
- total_prompt_tokens = 0
729
- total_completion_tokens = 0
730
- index = 0
731
-
732
- for gen, pt in zip(request.generate_responses, prompt_tokens_list):
733
- for choice in gen.choices:
734
- if not choice.token_ids:
735
- return self.create_error_response(
736
- f"choice {choice.index} in response {gen.request_id} "
737
- "has empty or null token_ids"
738
- )
739
- decoded_text = tokenizer.decode(
740
- choice.token_ids, skip_special_tokens=True
741
- )
742
- completion_logprobs = None
743
- if choice.logprobs is not None:
744
- resolved = _resolve_logprobs(choice.logprobs, tokenizer)
745
- completion_logprobs = _convert_chat_logprobs_to_completion_logprobs(
746
- resolved
747
- )
748
- choices.append(
749
- CompletionResponseChoice(
750
- index=index,
751
- text=decoded_text,
752
- finish_reason=choice.finish_reason,
753
- logprobs=completion_logprobs,
754
- )
755
- )
756
- total_completion_tokens += len(choice.token_ids)
757
- index += 1
758
- total_prompt_tokens += pt
759
-
276
if not request.generate_responses:
277
return self.create_error_response("generate_responses must not be empty")
278
@@ -794,175 +310,41 @@ class OpenAIServingRender:
310
kv_transfer_params=kv_params,
311
)
312
797
- def create_error_response(
798
- self,
799
- message: str | Exception,
800
- err_type: str = "BadRequestError",
801
- status_code: HTTPStatus = HTTPStatus.BAD_REQUEST,
802
- param: str | None = None,
803
- ) -> ErrorResponse:
804
- return create_error_response(message, err_type, status_code, param)
805
-
806
- async def _check_model(
807
- self,
808
- request: Any,
809
- ) -> ErrorResponse | None:
810
- return await self.model_registry.check_model(request.model)
313
+ @staticmethod
314
+ def _extract_mm_features(
315
+ engine_input: EngineInput,
316
+ ) -> MultiModalFeatures | None:
317
+ """Extract multimodal metadata from a rendered engine prompt.
318
812
- def validate_chat_template(
813
- self,
814
- request_chat_template: str | None,
815
- chat_template_kwargs: dict[str, Any] | None,
816
- trust_request_chat_template: bool,
817
- ) -> ErrorResponse | None:
818
- """Copied from OpenAIServing._validate_chat_template."""
819
- if not trust_request_chat_template and (
820
- request_chat_template is not None
821
- or (
822
- chat_template_kwargs
823
- and chat_template_kwargs.get("chat_template") is not None
824
- )
825
- ):
826
- return self.create_error_response(
827
- "Chat template is passed with request, but "
828
- "--trust-request-chat-template is not set. "
829
- "Refused request with untrusted chat template."
830
- )
831
- return None
319
+ Returns ``None`` for text-only prompts.
320
+ """
321
+ if engine_input.get("type") != "multimodal":
322
+ return None
323
833
- async def preprocess_completion(
834
- self,
835
- request: Any,
836
- prompt_input: str | list[str] | list[int] | list[list[int]] | None,
837
- prompt_embeds: bytes | list[bytes] | None,
838
- *,
839
- skip_mm_cache: bool = False,
840
- ) -> list[EngineInput]:
841
- """Copied from OpenAIServing._preprocess_completion."""
842
- prompts = list[SingletonPrompt | bytes]()
843
- if prompt_embeds is not None: # embeds take higher priority
844
- prompts.extend(prompt_to_seq(prompt_embeds))
845
- if prompt_input is not None:
846
- prompts.extend(prompt_to_seq(prompt_input))
847
- return await self.preprocess_cmpl(request, prompts, skip_mm_cache=skip_mm_cache)
848
-
849
- async def preprocess_cmpl(
850
- self,
851
- request: Any,
852
- prompts: Sequence[PromptType | bytes],
853
- *,
854
- skip_mm_cache: bool = False,
855
- ) -> list[EngineInput]:
856
- """Copied from OpenAIServing._preprocess_cmpl."""
857
- renderer = self.renderer
858
- model_config = self.model_config
859
-
860
- parsed_prompts = [
861
- (
862
- prompt
863
- if isinstance(prompt, bytes)
864
- else parse_model_prompt(model_config, prompt)
865
- )
866
- for prompt in prompts
867
- ]
868
- tok_params = request.build_tok_params(model_config)
869
-
870
- return await renderer.render_cmpl_async(
871
- parsed_prompts,
872
- tok_params,
873
- prompt_extras={
874
- k: v
875
- for k in ("mm_processor_kwargs", "cache_salt")
876
- if (v := getattr(request, k, None)) is not None
877
- },
878
- skip_mm_cache=skip_mm_cache,
879
- )
324
+ # At this point engine_input is a MultiModalInput TypedDict.
325
+ mm_engine_input = cast(MultiModalInput, engine_input)
326
+ mm_hashes: MultiModalHashes = mm_engine_input["mm_hashes"]
327
+ raw_placeholders: MultiModalPlaceholders = mm_engine_input["mm_placeholders"]
328
881
- async def preprocess_chat(
882
- self,
883
- request: Any,
884
- messages: list[Any],
885
- default_template: str | None,
886
- default_template_content_format: ChatTemplateContentFormatOption,
887
- default_template_kwargs: dict[str, Any] | None,
888
- tool_dicts: list[dict[str, Any]] | None = None,
889
- parser: type[Parser] | None = None,
890
- *,
891
- skip_mm_cache: bool = False,
892
- ) -> tuple[list[ConversationMessage], list[EngineInput]]:
893
- """Copied from OpenAIServing._preprocess_chat."""
894
- renderer = self.renderer
895
- mm_config = self.model_config.multimodal_config
896
-
897
- default_template_kwargs = merge_kwargs(
898
- default_template_kwargs,
899
- dict(
900
- tools=tool_dicts,
901
- tokenize=(
902
- is_mistral_tokenizer(renderer.tokenizer)
903
- or self.model_config.enable_prompt_embeds
904
- ),
905
- ),
906
- )
329
+ mm_placeholders = {
330
+ modality: [
331
+ PlaceholderRangeInfo(offset=p.offset, length=p.length) for p in ranges
332
+ ]
333
+ for modality, ranges in raw_placeholders.items()
334
+ }
335
908
- tok_params = request.build_tok_params(self.model_config)
909
- chat_params = request.build_chat_params(
910
- default_template, default_template_content_format
911
- ).with_defaults(
912
- default_template_kwargs,
913
- default_media_io_kwargs=(mm_config.media_io_kwargs if mm_config else None),
914
- default_mm_processor_kwargs=getattr(request, "mm_processor_kwargs", None),
915
- )
336
+ # Serialize tensor data per modality.
337
+ kwargs_data: dict[str, list[str | None]] | None = None
338
+ if raw_mm_kwargs := mm_engine_input.get("mm_kwargs"):
339
+ kwargs_data = {}
340
+ for modality, items in raw_mm_kwargs.items():
341
+ kwargs_data[modality] = [
342
+ encode_mm_kwargs_item(item) if item is not None else None
343
+ for item in items
344
+ ]
345
917
- (conversation,), (engine_input,) = await renderer.render_chat_async(
918
- [messages],
919
- chat_params,
920
- tok_params,
921
- prompt_extras={
922
- k: v
923
- for k in ("mm_processor_kwargs", "cache_salt")
924
- if (v := getattr(request, k, None)) is not None
925
- },
926
- skip_mm_cache=skip_mm_cache,
346
+ return MultiModalFeatures(
347
+ mm_hashes=mm_hashes,
348
+ mm_placeholders=mm_placeholders,
349
+ kwargs_data=kwargs_data,
350
)
928
-
929
- # tool parsing is done only if a tool_parser has been set and if
930
- # tool_choice is not "none" (if tool_choice is "none" but a tool_parser
931
- # is set, we want to prevent parsing a tool_call hallucinated by the LLM
932
- #
933
- # Exception: Mistral grammar-capable tokenizers always call
934
- # adjust_request — even for tool_choice="none" — so that the grammar
935
- # factory can prevent special-token leakage.
936
- if parser is not None:
937
- tokenizer = renderer.get_tokenizer()
938
- tool_parser = parser.tool_parser_cls
939
- tool_choice = getattr(request, "tool_choice", "none")
940
- is_mistral_grammar_eligible = (
941
- tool_parser is not None
942
- and is_mistral_tool_parser(tool_parser)
943
- and is_mistral_tokenizer(tokenizer)
944
- and tokenizer.supports_grammar
945
- )
946
- should_adjust_request = (
947
- parser.reasoning_parser_cls is not None
948
- or tool_choice != "none"
949
- or is_mistral_grammar_eligible
950
- )
951
- if should_adjust_request:
952
- if not isinstance(request, ChatCompletionRequest | ResponsesRequest):
953
- msg = (
954
- "Tool usage is only supported "
955
- "for Chat Completions API or Responses API requests, "
956
- f"but got {type(request).__name__}"
957
- )
958
- raise NotImplementedError(msg)
959
- request = parser(
960
- tokenizer,
961
- request.tools,
962
- model_config=self.model_config,
963
- chat_template_kwargs=chat_params.chat_template_kwargs,
964
- ).adjust_request(
965
- request=request,
966
- )
967
-
968
- return conversation, [engine_input]
vllm/entrypoints/serve/tokenize/serving.py
+9
-9
@@ -12,7 +12,6 @@ from vllm.entrypoints.openai.models.serving import (
12
OpenAIServingModels,
13
)
14
from vllm.entrypoints.serve.engine.serving import BaseServing
15
-from vllm.entrypoints.serve.render.serving import OpenAIServingRender
15
from vllm.entrypoints.serve.tokenize.protocol import (
16
DetokenizeRequest,
17
DetokenizeResponse,
@@ -24,6 +23,7 @@ from vllm.entrypoints.serve.tokenize.protocol import (
23
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
24
from vllm.inputs import TokensPrompt, tokens_input
25
from vllm.logger import init_logger
26
+from vllm.renderers.online_renderer import OnlineRenderer
27
from vllm.tokenizers import TokenizerLike
28
29
logger = init_logger(__name__)
@@ -33,22 +33,22 @@ class ServingTokenization(BaseServing):
33
def __init__(
34
self,
35
models: OpenAIServingModels | OpenAIModelRegistry,
36
- openai_serving_render: OpenAIServingRender,
36
+ online_renderer: OnlineRenderer,
37
*,
38
- request_logger: RequestLogger | None,
38
chat_template: str | None,
39
chat_template_content_format: ChatTemplateContentFormatOption,
40
default_chat_template_kwargs: dict[str, Any] | None = None,
41
trust_request_chat_template: bool = False,
42
+ request_logger: RequestLogger | None = None,
43
) -> None:
44
super().__init__(
45
models=models,
46
- model_config=openai_serving_render.model_config,
46
+ model_config=online_renderer.model_config,
47
request_logger=request_logger,
48
)
49
50
- self.renderer = openai_serving_render.renderer
51
- self.openai_serving_render = openai_serving_render
50
+ self.renderer = online_renderer.renderer
51
+ self.online_renderer = online_renderer
52
self.chat_template = chat_template
53
self.chat_template_content_format: Final = chat_template_content_format
54
self.default_chat_template_kwargs = default_chat_template_kwargs or {}
@@ -73,7 +73,7 @@ class ServingTokenization(BaseServing):
73
if request.tools is None
74
else [tool.model_dump() for tool in request.tools]
75
)
76
- error_check_ret = self.openai_serving_render.validate_chat_template(
76
+ error_check_ret = self.online_renderer.validate_chat_template(
77
request_chat_template=request.chat_template,
78
chat_template_kwargs=request.chat_template_kwargs,
79
trust_request_chat_template=self.trust_request_chat_template,
@@ -81,7 +81,7 @@ class ServingTokenization(BaseServing):
81
if error_check_ret is not None:
82
return error_check_ret
83
84
- _, engine_inputs = await self.openai_serving_render.preprocess_chat(
84
+ _, engine_inputs = await self.online_renderer.preprocess_chat(
85
request,
86
request.messages,
87
default_template=self.chat_template,
@@ -91,7 +91,7 @@ class ServingTokenization(BaseServing):
91
skip_mm_cache=True,
92
)
93
else:
94
- engine_inputs = await self.openai_serving_render.preprocess_completion(
94
+ engine_inputs = await self.online_renderer.preprocess_completion(
95
request,
96
prompt_input=request.prompt,
97
prompt_embeds=None,
vllm/renderers/online_derenderer.py
new
+334
@@ -0,0 +1,334 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+from typing import Any
4
+
5
+from vllm.config import ModelConfig
6
+from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
7
+from vllm.entrypoints.openai.chat_completion.protocol import (
8
+ ChatCompletionLogProbs,
9
+ ChatCompletionRequest,
10
+ ChatCompletionResponseChoice,
11
+ ChatMessage,
12
+)
13
+from vllm.entrypoints.openai.completion.protocol import (
14
+ CompletionLogProbs,
15
+ CompletionResponseChoice,
16
+)
17
+from vllm.entrypoints.openai.engine.protocol import ToolCall
18
+from vllm.entrypoints.openai.engine.serving import resolve_token_id_placeholder
19
+from vllm.entrypoints.serve.disagg.protocol import GenerateResponse
20
+from vllm.entrypoints.serve.utils.request_logger import RequestLogger
21
+from vllm.logger import init_logger
22
+from vllm.parser import Parser, ParserManager
23
+from vllm.renderers import BaseRenderer
24
+from vllm.tokenizers import TokenizerLike
25
+from vllm.utils import random_uuid
26
+
27
+logger = init_logger(__name__)
28
+
29
+
30
+class OnlineDerenderer:
31
+ def __init__(
32
+ self,
33
+ model_config: ModelConfig,
34
+ renderer: BaseRenderer,
35
+ *,
36
+ request_logger: RequestLogger | None,
37
+ chat_template: str | None,
38
+ chat_template_content_format: ChatTemplateContentFormatOption,
39
+ trust_request_chat_template: bool = False,
40
+ enable_auto_tools: bool = False,
41
+ exclude_tools_when_tool_choice_none: bool = False,
42
+ tool_parser: str | None = None,
43
+ reasoning_parser: str | None = None,
44
+ default_chat_template_kwargs: dict[str, Any] | None = None,
45
+ log_error_stack: bool = False,
46
+ ) -> None:
47
+ self.model_config = model_config
48
+ self.renderer = renderer
49
+ self.request_logger = request_logger
50
+
51
+ self.enable_auto_tools = enable_auto_tools
52
+ self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none
53
+ self.use_harmony = model_config.hf_config.model_type == "gpt_oss"
54
+ self.parser: type[Parser] | None = ParserManager.get_parser(
55
+ tool_parser_name=tool_parser,
56
+ reasoning_parser_name=reasoning_parser,
57
+ enable_auto_tools=enable_auto_tools,
58
+ model_name=model_config.model,
59
+ is_harmony=self.use_harmony,
60
+ )
61
+
62
+ self.chat_template = chat_template
63
+ self.chat_template_content_format: ChatTemplateContentFormatOption = (
64
+ chat_template_content_format
65
+ )
66
+ self.default_chat_template_kwargs: dict[str, Any] = (
67
+ default_chat_template_kwargs or {}
68
+ )
69
+ self.trust_request_chat_template = trust_request_chat_template
70
+
71
+ self.log_error_stack = log_error_stack
72
+ self.supports_browsing = False
73
+ self.supports_code_interpreter = False
74
+
75
+ async def derender_chat(
76
+ self,
77
+ generate_response: GenerateResponse,
78
+ chat_request: ChatCompletionRequest | None = None,
79
+ ) -> list[ChatCompletionResponseChoice]:
80
+ tokenizer = self.renderer.get_tokenizer()
81
+ choices: list[ChatCompletionResponseChoice] = []
82
+
83
+ for choice in generate_response.choices:
84
+ if not choice.token_ids:
85
+ raise ValueError(f"choice {choice.index} has empty or null token_ids")
86
+
87
+ resolved_logprobs = (
88
+ _resolve_logprobs(choice.logprobs, tokenizer)
89
+ if choice.logprobs is not None
90
+ else None
91
+ )
92
+
93
+ if self.parser is not None and chat_request is not None:
94
+ # Parser path: decode with special tokens preserved
95
+ # so the parser can see markers like </think>,
96
+ # <tool_call>, or Harmony channel tokens.
97
+ decoded_text = tokenizer.decode(
98
+ choice.token_ids, skip_special_tokens=False
99
+ )
100
+
101
+ chat_template_kwargs: dict[str, Any] = {}
102
+ if not self.use_harmony:
103
+ chat_template_kwargs = (
104
+ chat_request.build_chat_params(
105
+ self.chat_template,
106
+ self.chat_template_content_format,
107
+ )
108
+ .with_defaults(self.default_chat_template_kwargs)
109
+ .chat_template_kwargs
110
+ )
111
+
112
+ parser = self.parser(
113
+ tokenizer,
114
+ chat_request.tools,
115
+ chat_template_kwargs=chat_template_kwargs,
116
+ )
117
+ reasoning, content, tool_calls = parser.parse(
118
+ decoded_text,
119
+ chat_request,
120
+ enable_auto_tools=self.enable_auto_tools,
121
+ model_output_token_ids=choice.token_ids,
122
+ )
123
+
124
+ if not getattr(chat_request, "include_reasoning", True):
125
+ reasoning = None
126
+
127
+ tc_items = (
128
+ [
129
+ ToolCall(
130
+ id=random_uuid(),
131
+ function=tc,
132
+ )
133
+ for tc in tool_calls
134
+ ]
135
+ if tool_calls
136
+ else []
137
+ )
138
+
139
+ message = ChatMessage(
140
+ role="assistant",
141
+ reasoning=reasoning,
142
+ content=content,
143
+ tool_calls=tc_items,
144
+ )
145
+ else:
146
+ # No parser: plain detokenization.
147
+ decoded_text = tokenizer.decode(
148
+ choice.token_ids, skip_special_tokens=True
149
+ )
150
+ message = ChatMessage(role="assistant", content=decoded_text)
151
+
152
+ choices.append(
153
+ ChatCompletionResponseChoice(
154
+ index=choice.index,
155
+ message=message,
156
+ logprobs=resolved_logprobs,
157
+ finish_reason=choice.finish_reason,
158
+ )
159
+ )
160
+
161
+ return choices
162
+
163
+ async def derender_completion(
164
+ self,
165
+ generate_responses: list[GenerateResponse],
166
+ prompt_tokens: list[int] | None = None,
167
+ ) -> tuple[list[CompletionResponseChoice], int, int]:
168
+ n = len(generate_responses)
169
+ prompt_tokens_list: list[int] = (
170
+ prompt_tokens if prompt_tokens is not None else [0] * n
171
+ )
172
+
173
+ tokenizer = self.renderer.get_tokenizer()
174
+ choices: list[CompletionResponseChoice] = []
175
+ total_prompt_tokens = 0
176
+ total_completion_tokens = 0
177
+ index = 0
178
+
179
+ for gen, pt in zip(generate_responses, prompt_tokens_list):
180
+ for choice in gen.choices:
181
+ if not choice.token_ids:
182
+ raise ValueError(
183
+ f"choice {choice.index} in response {gen.request_id} "
184
+ "has empty or null token_ids"
185
+ )
186
+
187
+ decoded_text = tokenizer.decode(
188
+ choice.token_ids, skip_special_tokens=True
189
+ )
190
+ completion_logprobs = None
191
+ if choice.logprobs is not None:
192
+ resolved = _resolve_logprobs(choice.logprobs, tokenizer)
193
+ completion_logprobs = _convert_chat_logprobs_to_completion_logprobs(
194
+ resolved
195
+ )
196
+ choices.append(
197
+ CompletionResponseChoice(
198
+ index=index,
199
+ text=decoded_text,
200
+ finish_reason=choice.finish_reason,
201
+ logprobs=completion_logprobs,
202
+ )
203
+ )
204
+ total_completion_tokens += len(choice.token_ids)
205
+ index += 1
206
+ total_prompt_tokens += pt
207
+
208
+ return choices, total_prompt_tokens, total_completion_tokens
209
+
210
+
211
+def _parse_token_id_placeholder(token: str) -> int | None:
212
+ """Extract token ID from a 'token_id:N' placeholder string."""
213
+ if not token.startswith("token_id:"):
214
+ return None
215
+ try:
216
+ return int(token[len("token_id:") :])
217
+ except ValueError:
218
+ return None
219
+
220
+
221
+def _correct_decoded_token(
222
+ token_id: int, context_token_ids: list[int], tokenizer: TokenizerLike
223
+) -> str:
224
+ """Use preceding tokens as context to fix U+FFFD from byte-fallback.
225
+
226
+ Mirrors LogprobsProcessor._correct_decoded_token in v1/engine/logprobs.py.
227
+ """
228
+ max_ctx = min(len(context_token_ids), 4)
229
+
230
+ for num_ctx in range(1, max_ctx + 1):
231
+ context = context_token_ids[-num_ctx:]
232
+ full_decoded = tokenizer.decode(context + [token_id])
233
+
234
+ if full_decoded.endswith("�"):
235
+ continue
236
+
237
+ clean_end = len(context)
238
+ for j in range(len(context) - 1, -1, -1):
239
+ if tokenizer.decode([context[j]]).endswith("�"):
240
+ clean_end = j
241
+ else:
242
+ break
243
+
244
+ clean_prefix = tokenizer.decode(context[:clean_end]) if clean_end > 0 else ""
245
+
246
+ if full_decoded.startswith(clean_prefix):
247
+ return full_decoded[len(clean_prefix) :]
248
+
249
+ common_len = 0
250
+ for a, b in zip(clean_prefix, full_decoded):
251
+ if a != b:
252
+ break
253
+ common_len += 1
254
+ return full_decoded[common_len:]
255
+
256
+ return ""
257
+
258
+
259
+def _resolve_logprobs(
260
+ logprobs: ChatCompletionLogProbs, tokenizer: TokenizerLike
261
+) -> ChatCompletionLogProbs:
262
+ """Resolve token_id:N placeholders in a ChatCompletionLogProbs object."""
263
+ if logprobs.content is None:
264
+ return logprobs
265
+
266
+ context_token_ids: list[int] = []
267
+ resolved_content = []
268
+
269
+ for entry in logprobs.content:
270
+ token_str, token_bytes = resolve_token_id_placeholder(entry.token, tokenizer)
271
+ sampled_id = _parse_token_id_placeholder(entry.token)
272
+
273
+ if token_str.endswith("�") and sampled_id is not None:
274
+ token_str = _correct_decoded_token(sampled_id, context_token_ids, tokenizer)
275
+ token_bytes = list(token_str.encode("utf-8"))
276
+
277
+ resolved_top = []
278
+ for top in entry.top_logprobs:
279
+ top_str, top_bytes = resolve_token_id_placeholder(top.token, tokenizer)
280
+ top_id = _parse_token_id_placeholder(top.token)
281
+ if top_str.endswith("�") and top_id is not None:
282
+ top_str = _correct_decoded_token(top_id, context_token_ids, tokenizer)
283
+ top_bytes = list(top_str.encode("utf-8"))
284
+ resolved_top.append(
285
+ top.model_copy(update={"token": top_str, "bytes": top_bytes})
286
+ )
287
+
288
+ resolved_content.append(
289
+ entry.model_copy(
290
+ update={
291
+ "token": token_str,
292
+ "bytes": token_bytes,
293
+ "top_logprobs": resolved_top,
294
+ }
295
+ )
296
+ )
297
+
298
+ if sampled_id is not None:
299
+ context_token_ids.append(sampled_id)
300
+
301
+ return ChatCompletionLogProbs(content=resolved_content)
302
+
303
+
304
+def _convert_chat_logprobs_to_completion_logprobs(
305
+ logprobs: ChatCompletionLogProbs,
306
+) -> CompletionLogProbs:
307
+ """Convert ChatCompletionLogProbs (per-token objects) to CompletionLogProbs
308
+ (parallel flat lists) as required by the /v1/completions response schema."""
309
+ if logprobs.content is None:
310
+ return CompletionLogProbs()
311
+
312
+ tokens: list[str] = []
313
+ token_logprobs: list[float | None] = []
314
+ top_logprobs_list: list[dict[str, float] | None] = []
315
+ text_offset: list[int] = []
316
+
317
+ offset = 0
318
+ for entry in logprobs.content:
319
+ text_offset.append(offset)
320
+ tokens.append(entry.token)
321
+ token_logprobs.append(entry.logprob)
322
+ top_logprobs_list.append(
323
+ {t.token: t.logprob for t in entry.top_logprobs}
324
+ if entry.top_logprobs
325
+ else None
326
+ )
327
+ offset += len(entry.token)
328
+
329
+ return CompletionLogProbs(
330
+ text_offset=text_offset,
331
+ token_logprobs=token_logprobs,
332
+ tokens=tokens,
333
+ top_logprobs=top_logprobs_list,
334
+ )
vllm/renderers/online_renderer.py
new
+417
@@ -0,0 +1,417 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+from collections.abc import Sequence
4
+from http import HTTPStatus
5
+from typing import Any
6
+
7
+from openai_harmony import Message as OpenAIMessage
8
+
9
+from vllm.config import ModelConfig
10
+from vllm.entrypoints.chat_utils import (
11
+ ChatTemplateContentFormatOption,
12
+ ConversationMessage,
13
+)
14
+from vllm.entrypoints.openai.chat_completion.protocol import (
15
+ ChatCompletionRequest,
16
+)
17
+from vllm.entrypoints.openai.completion.protocol import (
18
+ CompletionRequest,
19
+)
20
+from vllm.entrypoints.openai.engine.protocol import ErrorResponse
21
+from vllm.entrypoints.openai.parser.harmony_utils import (
22
+ build_harmony_preamble,
23
+ extract_instructions_from_messages,
24
+ parse_chat_inputs_to_harmony_messages,
25
+ render_for_completion,
26
+)
27
+from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
28
+from vllm.entrypoints.serve.utils.error_response import create_error_response
29
+from vllm.entrypoints.serve.utils.request_logger import RequestLogger
30
+from vllm.inputs import (
31
+ EngineInput,
32
+ PromptType,
33
+ SingletonPrompt,
34
+ tokens_input,
35
+)
36
+from vllm.logger import init_logger
37
+from vllm.parser import Parser, ParserManager
38
+from vllm.renderers import BaseRenderer, merge_kwargs
39
+from vllm.renderers.inputs.preprocess import (
40
+ parse_model_prompt,
41
+ prompt_to_seq,
42
+)
43
+from vllm.utils.mistral import is_mistral_tokenizer, is_mistral_tool_parser
44
+from vllm.utils.mistral import mt as _mt
45
+
46
+logger = init_logger(__name__)
47
+
48
+
49
+class OnlineRenderer:
50
+ def __init__(
51
+ self,
52
+ model_config: ModelConfig,
53
+ renderer: BaseRenderer,
54
+ *,
55
+ request_logger: RequestLogger | None,
56
+ chat_template: str | None,
57
+ chat_template_content_format: ChatTemplateContentFormatOption,
58
+ trust_request_chat_template: bool = False,
59
+ enable_auto_tools: bool = False,
60
+ exclude_tools_when_tool_choice_none: bool = False,
61
+ tool_parser: str | None = None,
62
+ reasoning_parser: str | None = None,
63
+ default_chat_template_kwargs: dict[str, Any] | None = None,
64
+ log_error_stack: bool = False,
65
+ ) -> None:
66
+ self.model_config = model_config
67
+ self.renderer = renderer
68
+ self.request_logger = request_logger
69
+
70
+ self.enable_auto_tools = enable_auto_tools
71
+ self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none
72
+ self.use_harmony = model_config.hf_config.model_type == "gpt_oss"
73
+ self.parser: type[Parser] | None = ParserManager.get_parser(
74
+ tool_parser_name=tool_parser,
75
+ reasoning_parser_name=reasoning_parser,
76
+ enable_auto_tools=enable_auto_tools,
77
+ model_name=model_config.model,
78
+ is_harmony=self.use_harmony,
79
+ )
80
+
81
+ self.chat_template = chat_template
82
+ self.chat_template_content_format: ChatTemplateContentFormatOption = (
83
+ chat_template_content_format
84
+ )
85
+ self.default_chat_template_kwargs: dict[str, Any] = (
86
+ default_chat_template_kwargs or {}
87
+ )
88
+ self.trust_request_chat_template = trust_request_chat_template
89
+
90
+ self.log_error_stack = log_error_stack
91
+ self.supports_browsing = False
92
+ self.supports_code_interpreter = False
93
+
94
+ async def render_chat(
95
+ self,
96
+ request: ChatCompletionRequest,
97
+ *,
98
+ skip_mm_cache: bool = False,
99
+ ) -> tuple[list[ConversationMessage], list[EngineInput]] | ErrorResponse:
100
+ """Core preprocessing logic for chat requests (no model/engine check).
101
+
102
+ Called directly by render_chat_request and delegated to by
103
+ OpenAIServingChat.render_chat_request after its engine-aware checks.
104
+ """
105
+ tokenizer = self.renderer.tokenizer
106
+
107
+ tool_parser = self.parser.tool_parser_cls if self.parser is not None else None
108
+
109
+ if is_mistral_tokenizer(tokenizer):
110
+ # because of issues with pydantic we need to potentially
111
+ # re-serialize the tool_calls field of the request
112
+ _mt.maybe_serialize_tool_calls(request) # type: ignore[arg-type]
113
+ _mt.truncate_tool_call_ids(request) # type: ignore[arg-type]
114
+ _mt.validate_request_params(request)
115
+
116
+ # Check if tool parsing is unavailable (common condition)
117
+ tool_parsing_unavailable = (
118
+ tool_parser is None
119
+ and not is_mistral_tokenizer(tokenizer)
120
+ and not self.use_harmony
121
+ )
122
+
123
+ # Validate tool_choice when tool parsing is required but unavailable
124
+ if tool_parsing_unavailable and request.tool_choice not in (
125
+ None,
126
+ "none",
127
+ ):
128
+ if request.tool_choice == "auto" and not self.enable_auto_tools:
129
+ # for hf tokenizers, "auto" tools requires
130
+ # --enable-auto-tool-choice and --tool-call-parser
131
+ return self.create_error_response(
132
+ '"auto" tool choice requires '
133
+ "--enable-auto-tool-choice and --tool-call-parser to be set"
134
+ )
135
+ elif request.tool_choice != "auto":
136
+ # "required" or named tool requires tool parser
137
+ return self.create_error_response(
138
+ f'tool_choice="{request.tool_choice}" requires '
139
+ "--tool-call-parser to be set"
140
+ )
141
+
142
+ if request.tools is None or (
143
+ request.tool_choice == "none" and self.exclude_tools_when_tool_choice_none
144
+ ):
145
+ tool_dicts = None
146
+ else:
147
+ tool_dicts = [tool.model_dump() for tool in request.tools]
148
+
149
+ if not self.use_harmony:
150
+ # Common case.
151
+ error_check_ret = self.validate_chat_template(
152
+ request_chat_template=request.chat_template,
153
+ chat_template_kwargs=request.chat_template_kwargs,
154
+ trust_request_chat_template=self.trust_request_chat_template,
155
+ )
156
+ if error_check_ret is not None:
157
+ return error_check_ret
158
+
159
+ conversation, engine_inputs = await self.preprocess_chat(
160
+ request,
161
+ request.messages,
162
+ default_template=self.chat_template,
163
+ default_template_content_format=self.chat_template_content_format,
164
+ default_template_kwargs=self.default_chat_template_kwargs,
165
+ tool_dicts=tool_dicts,
166
+ parser=self.parser,
167
+ skip_mm_cache=skip_mm_cache,
168
+ )
169
+ else:
170
+ # For GPT-OSS.
171
+ should_include_tools = tool_dicts is not None
172
+ conversation, engine_inputs = self._make_request_with_harmony(
173
+ request, should_include_tools
174
+ )
175
+
176
+ return conversation, engine_inputs
177
+
178
+ def _make_request_with_harmony(
179
+ self,
180
+ request: ChatCompletionRequest,
181
+ should_include_tools: bool = True,
182
+ ):
183
+ """Build Harmony (GPT-OSS) messages and engine prompt from a chat request."""
184
+ messages: list[OpenAIMessage] = []
185
+
186
+ # because of issues with pydantic we need to potentially
187
+ # re-serialize the tool_calls field of the request
188
+ # for more info: see comment in `maybe_serialize_tool_calls`
189
+ _mt.maybe_serialize_tool_calls(request) # type: ignore[arg-type]
190
+
191
+ chat_messages = list(request.messages)
192
+ instructions, chat_messages = extract_instructions_from_messages(chat_messages)
193
+
194
+ # Add system message.
195
+ # NOTE: In Chat Completion API, browsing is enabled by default
196
+ # if the model supports it. TODO: Support browsing.
197
+ assert not self.supports_browsing
198
+ assert not self.supports_code_interpreter
199
+ if (reasoning_effort := request.reasoning_effort) == "none":
200
+ raise ValueError(f"Harmony does not support {reasoning_effort=}")
201
+ tools = request.tools if should_include_tools else None
202
+ messages.extend(
203
+ build_harmony_preamble(
204
+ instructions=instructions,
205
+ tools=tools, # type: ignore[arg-type]
206
+ reasoning_effort=reasoning_effort,
207
+ with_custom_tools=should_include_tools,
208
+ )
209
+ )
210
+
211
+ # Add remaining conversation messages.
212
+ messages.extend(parse_chat_inputs_to_harmony_messages(chat_messages))
213
+
214
+ # Render prompt token ids.
215
+ prompt_token_ids = render_for_completion(messages)
216
+ engine_input = tokens_input(prompt_token_ids, cache_salt=request.cache_salt)
217
+
218
+ return messages, [engine_input]
219
+
220
+ async def render_completion(
221
+ self,
222
+ request: CompletionRequest,
223
+ *,
224
+ skip_mm_cache: bool = False,
225
+ ) -> list[EngineInput] | ErrorResponse:
226
+ """Core preprocessing logic for completion requests (no model/engine check).
227
+
228
+ Called directly by render_completion_request and delegated to by
229
+ OpenAIServingCompletion.render_completion_request after its engine-aware checks.
230
+ """
231
+ # Return error for unsupported features.
232
+ if request.suffix is not None:
233
+ return self.create_error_response("suffix is not currently supported")
234
+
235
+ if request.echo and request.prompt_embeds is not None:
236
+ return self.create_error_response("Echo is unsupported with prompt embeds.")
237
+
238
+ if request.prompt_logprobs is not None and request.prompt_embeds is not None:
239
+ return self.create_error_response(
240
+ "prompt_logprobs is not compatible with prompt embeds."
241
+ )
242
+
243
+ engine_inputs = await self.preprocess_completion(
244
+ request,
245
+ prompt_input=request.prompt,
246
+ prompt_embeds=request.prompt_embeds,
247
+ skip_mm_cache=skip_mm_cache,
248
+ )
249
+
250
+ return engine_inputs
251
+
252
+ def create_error_response(
253
+ self,
254
+ message: str | Exception,
255
+ err_type: str = "BadRequestError",
256
+ status_code: HTTPStatus = HTTPStatus.BAD_REQUEST,
257
+ param: str | None = None,
258
+ ) -> ErrorResponse:
259
+ return create_error_response(message, err_type, status_code, param)
260
+
261
+ def validate_chat_template(
262
+ self,
263
+ request_chat_template: str | None,
264
+ chat_template_kwargs: dict[str, Any] | None,
265
+ trust_request_chat_template: bool,
266
+ ) -> ErrorResponse | None:
267
+ """Copied from OpenAIServing._validate_chat_template."""
268
+ if not trust_request_chat_template and (
269
+ request_chat_template is not None
270
+ or (
271
+ chat_template_kwargs
272
+ and chat_template_kwargs.get("chat_template") is not None
273
+ )
274
+ ):
275
+ return self.create_error_response(
276
+ "Chat template is passed with request, but "
277
+ "--trust-request-chat-template is not set. "
278
+ "Refused request with untrusted chat template."
279
+ )
280
+ return None
281
+
282
+ async def preprocess_completion(
283
+ self,
284
+ request: Any,
285
+ prompt_input: str | list[str] | list[int] | list[list[int]] | None,
286
+ prompt_embeds: bytes | list[bytes] | None,
287
+ *,
288
+ skip_mm_cache: bool = False,
289
+ ) -> list[EngineInput]:
290
+ """Copied from OpenAIServing._preprocess_completion."""
291
+ prompts = list[SingletonPrompt | bytes]()
292
+ if prompt_embeds is not None: # embeds take higher priority
293
+ prompts.extend(prompt_to_seq(prompt_embeds))
294
+ if prompt_input is not None:
295
+ prompts.extend(prompt_to_seq(prompt_input))
296
+ return await self.preprocess_cmpl(request, prompts, skip_mm_cache=skip_mm_cache)
297
+
298
+ async def preprocess_cmpl(
299
+ self,
300
+ request: Any,
301
+ prompts: Sequence[PromptType | bytes],
302
+ *,
303
+ skip_mm_cache: bool = False,
304
+ ) -> list[EngineInput]:
305
+ """Copied from OpenAIServing._preprocess_cmpl."""
306
+ renderer = self.renderer
307
+ model_config = self.model_config
308
+
309
+ parsed_prompts = [
310
+ (
311
+ prompt
312
+ if isinstance(prompt, bytes)
313
+ else parse_model_prompt(model_config, prompt)
314
+ )
315
+ for prompt in prompts
316
+ ]
317
+ tok_params = request.build_tok_params(model_config)
318
+
319
+ return await renderer.render_cmpl_async(
320
+ parsed_prompts,
321
+ tok_params,
322
+ prompt_extras={
323
+ k: v
324
+ for k in ("mm_processor_kwargs", "cache_salt")
325
+ if (v := getattr(request, k, None)) is not None
326
+ },
327
+ skip_mm_cache=skip_mm_cache,
328
+ )
329
+
330
+ async def preprocess_chat(
331
+ self,
332
+ request: Any,
333
+ messages: list[Any],
334
+ default_template: str | None,
335
+ default_template_content_format: ChatTemplateContentFormatOption,
336
+ default_template_kwargs: dict[str, Any] | None,
337
+ tool_dicts: list[dict[str, Any]] | None = None,
338
+ parser: type[Parser] | None = None,
339
+ *,
340
+ skip_mm_cache: bool = False,
341
+ ) -> tuple[list[ConversationMessage], list[EngineInput]]:
342
+ """Copied from OpenAIServing._preprocess_chat."""
343
+ renderer = self.renderer
344
+ mm_config = self.model_config.multimodal_config
345
+
346
+ default_template_kwargs = merge_kwargs(
347
+ default_template_kwargs,
348
+ dict(
349
+ tools=tool_dicts,
350
+ tokenize=(
351
+ is_mistral_tokenizer(renderer.tokenizer)
352
+ or self.model_config.enable_prompt_embeds
353
+ ),
354
+ ),
355
+ )
356
+
357
+ tok_params = request.build_tok_params(self.model_config)
358
+ chat_params = request.build_chat_params(
359
+ default_template, default_template_content_format
360
+ ).with_defaults(
361
+ default_template_kwargs,
362
+ default_media_io_kwargs=(mm_config.media_io_kwargs if mm_config else None),
363
+ default_mm_processor_kwargs=getattr(request, "mm_processor_kwargs", None),
364
+ )
365
+
366
+ (conversation,), (engine_input,) = await renderer.render_chat_async(
367
+ [messages],
368
+ chat_params,
369
+ tok_params,
370
+ prompt_extras={
371
+ k: v
372
+ for k in ("mm_processor_kwargs", "cache_salt")
373
+ if (v := getattr(request, k, None)) is not None
374
+ },
375
+ skip_mm_cache=skip_mm_cache,
376
+ )
377
+
378
+ # tool parsing is done only if a tool_parser has been set and if
379
+ # tool_choice is not "none" (if tool_choice is "none" but a tool_parser
380
+ # is set, we want to prevent parsing a tool_call hallucinated by the LLM
381
+ #
382
+ # Exception: Mistral grammar-capable tokenizers always call
383
+ # adjust_request — even for tool_choice="none" — so that the grammar
384
+ # factory can prevent special-token leakage.
385
+ if parser is not None:
386
+ tokenizer = renderer.get_tokenizer()
387
+ tool_parser = parser.tool_parser_cls
388
+ tool_choice = getattr(request, "tool_choice", "none")
389
+ is_mistral_grammar_eligible = (
390
+ tool_parser is not None
391
+ and is_mistral_tool_parser(tool_parser)
392
+ and is_mistral_tokenizer(tokenizer)
393
+ and tokenizer.supports_grammar
394
+ )
395
+ should_adjust_request = (
396
+ parser.reasoning_parser_cls is not None
397
+ or tool_choice != "none"
398
+ or is_mistral_grammar_eligible
399
+ )
400
+ if should_adjust_request:
401
+ if not isinstance(request, ChatCompletionRequest | ResponsesRequest):
402
+ msg = (
403
+ "Tool usage is only supported "
404
+ "for Chat Completions API or Responses API requests, "
405
+ f"but got {type(request).__name__}"
406
+ )
407
+ raise NotImplementedError(msg)
408
+ request = parser(
409
+ tokenizer,
410
+ request.tools,
411
+ model_config=self.model_config,
412
+ chat_template_kwargs=chat_params.chat_template_kwargs,
413
+ ).adjust_request(
414
+ request=request,
415
+ )
416
+
417
+ return conversation, [engine_input]