@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 6a2a2bb02b

[Frontend] Add stateless /v1/responses/render endpoint (#50195)

Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> Co-authored-by: OpenAI Codex <codex@openai.com>

Francisco Javier Arceo committed Sep 7, 2026 at 08:50 UTC 6a2a2bb02b563b83f946012959fd3927984d072a
19 files changed +1228 -298
docs/serving/online_serving/README.md
+2
@@ -141,6 +141,8 @@ For further details on renderer APIs, please refer to [this page](renderer.md).
141 - Render completion requests
142 - [Chat Completions Render API](renderer.md) (`/v1/chat/completions/render`)
143 - Render chat completions
144 +- [Responses Render API](renderer.md) (`/v1/responses/render`)
145 + - Render self-contained Responses requests
146
147 ### Derenderer APIs
148
docs/serving/online_serving/renderer.md
+23
@@ -25,6 +25,29 @@ VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1 vllm serve <model>
25 - Render completion requests
26 - [Chat Completions Render API](renderer.md) (`/v1/chat/completions/render`)
27 - Render chat completions
28 +- [Responses Render API](renderer.md) (`/v1/responses/render`)
29 + - Render a self-contained Responses request
30 +
31 +The Responses render endpoint uses the same prompt construction as
32 +`/v1/responses` and returns one token-in `GenerateRequest`. It is stateless:
33 +inline history is supported, but `previous_response_id` is not. Callers must
34 +resolve stored response state and include the resulting history in the request
35 +before rendering.
36 +
37 +For multimodal requests, the `GenerateRequest` contains the model-processed
38 +multimodal payload, which can be substantially larger than the source image or
39 +video. The caller must forward that payload unchanged to the generation service
40 +and provision transport limits and memory accordingly.
41 +
42 +```bash
43 +curl http://localhost:8000/v1/responses/render \
44 + -H "Content-Type: application/json" \
45 + -d '{
46 + "model": "meta-llama/Llama-3.1-8B-Instruct",
47 + "input": "Explain prefix caching in one sentence.",
48 + "max_output_tokens": 32
49 + }'
50 +```
51
52 For the post processing counterpart that turns generated token IDs back into OpenAI compatible responses, see the [Derenderer APIs](derenderer.md).
53
docs/usage/security.md
+1
@@ -170,6 +170,7 @@ When `--api-key` is configured, the following `/v1` endpoints require Bearer tok
170 - `/v1/messages/render` - Render Anthropic-compatible messages (available on `vllm serve` only when `VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1`, or on `vllm launch render` unless explicitly disabled)
171 - `/v1/messages/count_tokens` - Count tokens for Anthropic messages
172 - `/v1/responses` - Create a response
173 +- `/v1/responses/render` - Render a self-contained response request (available on `vllm serve` only when `VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1`, or on `vllm launch render` unless explicitly disabled)
174 - `/v1/responses/{response_id}` - Retrieve a response
175 - `/v1/responses/{response_id}/cancel` - Cancel a response
176 - `/v1/score` - Scoring API
tests/entrypoints/launchers/test_cli_args.py
+22
@@ -2,9 +2,11 @@
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 import json
5 +from types import SimpleNamespace
6
7 import pytest
8
9 +import vllm.entrypoints.launchers.cli_args as cli_args_module
10 from tests.utils import VLLM_PATH
11 from vllm.entrypoints.launchers.cli_args import (
12 make_arg_parser,
@@ -332,6 +334,26 @@ def test_default_chat_template_kwargs_default_none(serve_parser):
334 assert args.default_chat_template_kwargs is None
335
336
337 +@pytest.mark.parametrize(
338 + "default_kwargs,cohere_format,expected",
339 + [
340 + (None, "cmd3", {"cohere_format": "cmd3"}),
341 + (
342 + {"cohere_format": "cmd4", "custom": True},
343 + "cmd3",
344 + {"cohere_format": "cmd4", "custom": True},
345 + ),
346 + ],
347 +)
348 +def test_resolve_default_chat_template_kwargs(default_kwargs, cohere_format, expected):
349 + args = SimpleNamespace(
350 + default_chat_template_kwargs=default_kwargs,
351 + cohere_format=cohere_format,
352 + )
353 +
354 + assert cli_args_module.resolve_default_chat_template_kwargs(args) == expected
355 +
356 +
357 def test_default_chat_template_kwargs_invalid_json(serve_parser):
358 """Ensure invalid JSON raises an error"""
359 with pytest.raises(SystemExit):
tests/entrypoints/openai/responses/test_mcp_tools.py
+24
@@ -4,14 +4,19 @@
4
5 from __future__ import annotations
6
7 +from types import SimpleNamespace
8 +from unittest.mock import AsyncMock, MagicMock
9 +
10 import pytest
11 import pytest_asyncio
12 from openai import OpenAI
13 from openai_harmony import Message, ToolDescription, ToolNamespaceConfig
14
15 +import vllm.entrypoints.mcp.tool_server as tool_server_module
16 from tests.utils import RemoteOpenAIServer
17 from vllm.entrypoints.mcp.tool_server import (
18 MCPToolServer,
19 + init_tool_server,
20 post_process_tools_description,
21 )
22
@@ -39,6 +44,25 @@ _PYTHON_TOOL_INSTRUCTION = (
44 )
45
46
47 +@pytest.mark.asyncio
48 +async def test_init_tool_server_without_configuration():
49 + args = SimpleNamespace(tool_server=None)
50 +
51 + assert await init_tool_server(args) is None
52 +
53 +
54 +@pytest.mark.asyncio
55 +async def test_init_tool_server_with_mcp_url(monkeypatch: pytest.MonkeyPatch):
56 + server = MagicMock()
57 + server.add_tool_server = AsyncMock()
58 + monkeypatch.setattr(tool_server_module, "MCPToolServer", lambda: server)
59 +
60 + result = await init_tool_server(SimpleNamespace(tool_server="http://mcp.test"))
61 +
62 + assert result is server
63 + server.add_tool_server.assert_awaited_once_with("http://mcp.test")
64 +
65 +
66 @pytest.mark.skip_global_cleanup
67 class TestMCPToolServerUnit:
68 """Test MCP tool description processing and filtering.
tests/entrypoints/openai/responses/test_serving_responses.py
+479 -3
@@ -2,12 +2,16 @@
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 from contextlib import AsyncExitStack
5 -from unittest.mock import MagicMock
5 +from types import SimpleNamespace
6 +from unittest.mock import AsyncMock, MagicMock
7
8 import pytest
9 import pytest_asyncio
10 from openai.types.responses import (
11 + ResponseFunctionToolCall,
12 ResponseOutputItemDoneEvent,
13 + ResponseOutputMessage,
14 + ResponseOutputText,
15 ResponseReasoningItem,
16 ResponseReasoningTextDeltaEvent,
17 ResponseReasoningTextDoneEvent,
@@ -23,6 +27,8 @@ from openai.types.responses.tool import (
27 Mcp,
28 Tool,
29 )
30 +from openai_harmony import Message as OpenAIHarmonyMessage
31 +from openai_harmony import Role
32
33 import vllm.envs as envs
34 from vllm.entrypoints.generate.base.protocol import (
@@ -32,7 +38,11 @@ from vllm.entrypoints.generate.base.protocol import (
38 RequestResponseMetadata,
39 )
40 from vllm.entrypoints.mcp.tool_server import ToolServer
35 -from vllm.entrypoints.openai.responses.context import ConversationContext, SimpleContext
41 +from vllm.entrypoints.openai.responses.context import (
42 + ConversationContext,
43 + HarmonyContext,
44 + SimpleContext,
45 +)
46 from vllm.entrypoints.openai.responses.protocol import (
47 ResponseCreatedEvent,
48 ResponseRawMessageAndToken,
@@ -42,7 +52,6 @@ from vllm.entrypoints.openai.responses.protocol import (
52 )
53 from vllm.entrypoints.openai.responses.serving import (
54 OpenAIServingResponses,
45 - _extract_allowed_tools_from_mcp_requests,
55 extract_tool_types,
56 )
57 from vllm.entrypoints.openai.responses.streaming_events import (
@@ -52,8 +61,23 @@ from vllm.entrypoints.serve.engine.protocol import ErrorResponse
61 from vllm.inputs import tokens_input
62 from vllm.outputs import CompletionOutput, RequestOutput
63 from vllm.parser.harmony import Segment
64 +from vllm.renderers import TokenizeParams
65 +from vllm.renderers.online_renderer import (
66 + OnlineRenderer,
67 + _extract_allowed_tools_from_mcp_requests,
68 +)
69 from vllm.sampling_params import SamplingParams
70
71 +pytestmark = pytest.mark.skip_global_cleanup
72 +
73 +
74 +def _new_online_renderer() -> OnlineRenderer:
75 + renderer = OnlineRenderer.__new__(OnlineRenderer)
76 + renderer.exclude_tools_when_tool_choice_none = False
77 + renderer.parser = None
78 + renderer.trust_request_chat_template = True
79 + return renderer
80 +
81
82 class MockConversationContext(ConversationContext):
83 """Mock conversation context for testing"""
@@ -205,6 +229,325 @@ def test_response_created_event_uses_public_json_schema_alias() -> None:
229 assert event.response.text.format.model_dump(by_alias=True)["schema"] == schema
230
231
232 +@pytest.mark.asyncio
233 +async def test_online_renderer_renders_non_harmony_responses_with_explicit_history():
234 + renderer = _new_online_renderer()
235 + renderer.use_harmony = False
236 + renderer.chat_template = "server-template"
237 + renderer.chat_template_content_format = "string"
238 + renderer.default_chat_template_kwargs = {"server_default": "kept"}
239 + renderer.parser = None
240 + renderer.preprocess_chat = AsyncMock(
241 + return_value=(
242 + [],
243 + [tokens_input([11, 12, 13], cache_salt="request-salt")],
244 + )
245 + )
246 + previous_messages = [
247 + {"role": "system", "content": "old instructions"},
248 + {"role": "user", "content": "first"},
249 + ]
250 + previous_outputs = [
251 + ResponseOutputMessage(
252 + id="msg_1",
253 + content=[
254 + ResponseOutputText(
255 + annotations=[],
256 + text="prior answer",
257 + type="output_text",
258 + )
259 + ],
260 + role="assistant",
261 + status="completed",
262 + type="message",
263 + )
264 + ]
265 + request = ResponsesRequest(
266 + input="next",
267 + instructions="new instructions",
268 + tools=[],
269 + cache_salt="request-salt",
270 + chat_template_kwargs={"request_default": "kept"},
271 + )
272 +
273 + result = await renderer.render_responses(
274 + request,
275 + previous_messages=previous_messages,
276 + previous_response_outputs=previous_outputs,
277 + )
278 +
279 + assert not isinstance(result, ErrorResponse)
280 + assert result.messages == [
281 + {"role": "system", "content": "new instructions"},
282 + {"role": "user", "content": "first"},
283 + {"role": "assistant", "content": "prior answer"},
284 + {"role": "user", "content": "next"},
285 + ]
286 + assert result.engine_input["prompt_token_ids"] == [11, 12, 13]
287 + assert result.engine_input["cache_salt"] == "request-salt"
288 + assert previous_messages[0]["content"] == "old instructions"
289 + assert previous_outputs[0].content[0].text == "prior answer"
290 +
291 + preprocess_kwargs = renderer.preprocess_chat.call_args.kwargs
292 + assert preprocess_kwargs["default_template"] == "server-template"
293 + assert preprocess_kwargs["default_template_content_format"] == "string"
294 + assert preprocess_kwargs["default_template_kwargs"]["server_default"] == "kept"
295 + assert preprocess_kwargs["default_template_kwargs"]["request_default"] == "kept"
296 +
297 +
298 +@pytest.mark.asyncio
299 +async def test_online_renderer_rejects_untrusted_responses_chat_template():
300 + renderer = _new_online_renderer()
301 + renderer.use_harmony = False
302 + renderer.chat_template = "server-template"
303 + renderer.chat_template_content_format = "string"
304 + renderer.default_chat_template_kwargs = {}
305 + renderer.exclude_tools_when_tool_choice_none = False
306 + renderer.trust_request_chat_template = False
307 + renderer.parser = None
308 + renderer.preprocess_chat = AsyncMock(
309 + return_value=([], [tokens_input([1])]),
310 + )
311 + request = ResponsesRequest(
312 + input="hello",
313 + chat_template_kwargs={"chat_template": "{{ messages }}"},
314 + )
315 +
316 + result = await renderer.render_responses(request)
317 +
318 + assert isinstance(result, ErrorResponse)
319 + assert result.error.code == 400
320 + assert "untrusted chat template" in result.error.message.lower()
321 + renderer.preprocess_chat.assert_not_awaited()
322 +
323 +
324 +@pytest.mark.asyncio
325 +async def test_online_renderer_rejects_harmony_history_for_non_harmony_model():
326 + renderer = _new_online_renderer()
327 + renderer.use_harmony = False
328 + request = ResponsesRequest(input="next")
329 +
330 + result = await renderer.render_responses(
331 + request,
332 + previous_messages=[
333 + OpenAIHarmonyMessage.from_role_and_content(Role.USER, "first")
334 + ],
335 + )
336 +
337 + assert isinstance(result, ErrorResponse)
338 + assert result.error.param == "previous_response_id"
339 +
340 +
341 +@pytest.mark.asyncio
342 +async def test_online_renderer_rejects_chat_history_for_harmony_model():
343 + renderer = _new_online_renderer()
344 + renderer.use_harmony = True
345 + request = ResponsesRequest(input="next")
346 +
347 + result = await renderer.render_responses(
348 + request,
349 + previous_messages=[{"role": "user", "content": "first"}],
350 + )
351 +
352 + assert isinstance(result, ErrorResponse)
353 + assert result.error.param == "previous_response_id"
354 +
355 +
356 +@pytest.mark.asyncio
357 +async def test_online_renderer_adjusts_harmony_tool_choice(
358 + monkeypatch: pytest.MonkeyPatch,
359 +):
360 + class StubParser:
361 + def __init__(self, tokenizer, tools, *, model_config):
362 + pass
363 +
364 + def adjust_request(self, request):
365 + request.cache_salt = "adjusted"
366 + return request
367 +
368 + renderer = _new_online_renderer()
369 + renderer.use_harmony = True
370 + renderer.parser = StubParser
371 + renderer.model_config = SimpleNamespace(max_model_len=100)
372 + tokenizer = SimpleNamespace(truncation_side="left")
373 + renderer.renderer = SimpleNamespace(
374 + tokenizer=tokenizer,
375 + get_tokenizer=lambda: tokenizer,
376 + )
377 + monkeypatch.setattr(
378 + "vllm.renderers.online_renderer.render_for_completion",
379 + lambda messages: [1],
380 + )
381 + request = ResponsesRequest(
382 + input="next",
383 + tools=[
384 + {
385 + "type": "function",
386 + "name": "lookup",
387 + "parameters": {"type": "object", "properties": {}},
388 + }
389 + ],
390 + tool_choice={"type": "function", "name": "lookup"},
391 + )
392 +
393 + result = await renderer.render_responses(request)
394 +
395 + assert not isinstance(result, ErrorResponse)
396 + assert result.engine_input["cache_salt"] == "adjusted"
397 +
398 +
399 +@pytest.mark.asyncio
400 +async def test_online_renderer_applies_responses_token_budget_to_harmony_prompt(
401 + monkeypatch: pytest.MonkeyPatch,
402 +):
403 + renderer = _new_online_renderer()
404 + renderer.use_harmony = True
405 + renderer.model_config = SimpleNamespace(max_model_len=5)
406 + renderer.renderer = SimpleNamespace(
407 + tokenizer=SimpleNamespace(truncation_side="left")
408 + )
409 + monkeypatch.setattr(
410 + "vllm.renderers.online_renderer.render_for_completion",
411 + lambda messages: [1, 2, 3, 4, 5, 6],
412 + )
413 + request = ResponsesRequest(
414 + input="hello",
415 + max_output_tokens=2,
416 + truncation="auto",
417 + )
418 +
419 + result = await renderer.render_responses(request)
420 +
421 + assert not isinstance(result, ErrorResponse)
422 + assert result.engine_input["prompt_token_ids"] == [4, 5, 6]
423 +
424 +
425 +@pytest.mark.asyncio
426 +async def test_online_renderer_renders_harmony_continuation_with_call_linkage(
427 + monkeypatch: pytest.MonkeyPatch,
428 +):
429 + renderer = _new_online_renderer()
430 + renderer.use_harmony = True
431 + renderer.model_config = SimpleNamespace(max_model_len=100)
432 + renderer.renderer = SimpleNamespace(
433 + tokenizer=SimpleNamespace(truncation_side="left")
434 + )
435 + monkeypatch.setattr(
436 + "vllm.renderers.online_renderer.render_for_completion",
437 + lambda messages: [1],
438 + )
439 + previous_messages = [
440 + OpenAIHarmonyMessage.from_role_and_content(Role.USER, "first"),
441 + ]
442 + previous_outputs = [
443 + ResponseFunctionToolCall(
444 + arguments='{"city":"Paris"}',
445 + call_id="call_1",
446 + name="lookup_weather",
447 + type="function_call",
448 + )
449 + ]
450 + request = ResponsesRequest(
451 + input=[
452 + {
453 + "type": "function_call_output",
454 + "call_id": "call_1",
455 + "output": "sunny",
456 + }
457 + ],
458 + instructions="replacement instructions are ignored on continuations",
459 + reasoning={"effort": "high"},
460 + tools=[
461 + {
462 + "type": "function",
463 + "name": "lookup_weather",
464 + "parameters": {"type": "object", "properties": {}},
465 + }
466 + ],
467 + cache_salt="request-salt",
468 + )
469 +
470 + result = await renderer.render_responses(
471 + request,
472 + previous_messages=previous_messages,
473 + previous_response_outputs=previous_outputs,
474 + )
475 +
476 + assert not isinstance(result, ErrorResponse)
477 + assert result.messages[0] is previous_messages[0]
478 + tool_output = result.messages[-1]
479 + assert tool_output.author.role == "tool"
480 + assert tool_output.author.name == "functions.lookup_weather"
481 + assert tool_output.channel == "commentary"
482 + assert tool_output.recipient == "assistant"
483 + assert result.engine_input["cache_salt"] == "request-salt"
484 + assert "arrival_time" in result.engine_input
485 + assert previous_messages == [
486 + OpenAIHarmonyMessage.from_role_and_content(Role.USER, "first"),
487 + ]
488 + assert previous_outputs[0].call_id == "call_1"
489 +
490 +
491 +@pytest.mark.asyncio
492 +async def test_online_renderer_returns_typed_error_for_bad_harmony_call_reference():
493 + renderer = _new_online_renderer()
494 + renderer.use_harmony = True
495 + request = ResponsesRequest(
496 + input=[
497 + {
498 + "type": "function_call_output",
499 + "call_id": "missing",
500 + "output": "orphaned",
501 + }
502 + ],
503 + tools=[],
504 + )
505 +
506 + result = await renderer.render_responses(request)
507 +
508 + assert isinstance(result, ErrorResponse)
509 + assert result.error.type == "invalid_request_error"
510 + assert result.error.param == "input"
511 + assert "No call message found for missing" in result.error.message
512 +
513 +
514 +@pytest.mark.asyncio
515 +async def test_online_renderer_preserves_missing_harmony_builtin_tool_behavior(
516 + monkeypatch: pytest.MonkeyPatch,
517 +):
518 + renderer = _new_online_renderer()
519 + renderer.use_harmony = True
520 + renderer.model_config = SimpleNamespace(max_model_len=100)
521 + renderer.renderer = SimpleNamespace(
522 + tokenizer=SimpleNamespace(truncation_side="left")
523 + )
524 + monkeypatch.setattr(
525 + "vllm.renderers.online_renderer.render_for_completion",
526 + lambda messages: [1],
527 + )
528 + request = ResponsesRequest(
529 + input="search",
530 + tools=[{"type": "web_search_preview"}],
531 + )
532 +
533 + result = await renderer.render_responses(request, tool_server=None)
534 +
535 + assert not isinstance(result, ErrorResponse)
536 +
537 +
538 +@pytest.mark.parametrize(
539 + "engine_inputs",
540 + [[], [tokens_input([1]), tokens_input([2])]],
541 +)
542 +def test_responses_render_result_rejects_non_single_prompt(engine_inputs):
543 + renderer = _new_online_renderer()
544 +
545 + result = renderer._responses_render_result([], engine_inputs)
546 +
547 + assert isinstance(result, ErrorResponse)
548 + assert f"got {len(engine_inputs)}" in result.error.message
549 +
550 +
551 class TestInitializeToolSessions:
552 """Test class for _initialize_tool_sessions method"""
553
@@ -240,6 +583,139 @@ class TestInitializeToolSessions:
583
584 return instance
585
586 + @pytest.mark.asyncio
587 + async def test_stateful_harmony_delegates_named_tool_choice_to_renderer(
588 + self,
589 + serving_responses_instance,
590 + ):
591 + serving_responses_instance.use_harmony = True
592 + serving_responses_instance.online_renderer.render_responses = AsyncMock(
593 + return_value=SimpleNamespace(
594 + messages=[],
595 + engine_input=tokens_input([1]),
596 + )
597 + )
598 + request = ResponsesRequest(
599 + input="hello",
600 + tool_choice={"type": "function", "name": "lookup"},
601 + tools=[
602 + {
603 + "type": "function",
604 + "name": "lookup",
605 + "parameters": {"type": "object", "properties": {}},
606 + }
607 + ],
608 + )
609 +
610 + result = await serving_responses_instance._render_resolved_response_inputs(
611 + request,
612 + None,
613 + )
614 +
615 + assert result.engine_input["prompt_token_ids"] == [1]
616 + serving_responses_instance.online_renderer.render_responses.assert_awaited_once()
617 +
618 + @pytest.mark.asyncio
619 + async def test_render_next_turn_reuses_shared_responses_renderer(
620 + self, serving_responses_instance
621 + ):
622 + serving_responses_instance.online_renderer.render_responses = AsyncMock(
623 + return_value=SimpleNamespace(
624 + messages=[],
625 + engine_input=tokens_input([8, 9], cache_salt="request-salt"),
626 + )
627 + )
628 + serving_responses_instance.online_renderer.preprocess_chat = AsyncMock(
629 + return_value=([], [tokens_input([1])])
630 + )
631 + request = ResponsesRequest(
632 + input="first",
633 + instructions="initial instructions",
634 + cache_salt="request-salt",
635 + )
636 + turn_messages = [
637 + {"role": "user", "content": "first"},
638 + {"role": "assistant", "content": "tool call"},
639 + {"role": "tool", "content": "tool output"},
640 + ]
641 +
642 + engine_inputs = await serving_responses_instance._render_next_turn(
643 + request,
644 + turn_messages,
645 + )
646 +
647 + assert engine_inputs[0]["prompt_token_ids"] == [8, 9]
648 + render_request = (
649 + serving_responses_instance.online_renderer.render_responses.call_args.args[
650 + 0
651 + ]
652 + )
653 + assert render_request.input == turn_messages
654 + assert render_request.instructions is None
655 + assert render_request.cache_salt == "request-salt"
656 +
657 + @pytest.mark.asyncio
658 + async def test_harmony_tool_followup_preserves_render_params(
659 + self, serving_responses_instance, monkeypatch: pytest.MonkeyPatch
660 + ):
661 + class ToolCallingHarmonyContext(HarmonyContext):
662 + def __init__(self):
663 + self._messages = []
664 + self._needs_tool_call = True
665 +
666 + def append_output(self, output) -> None:
667 + pass
668 +
669 + def need_builtin_tool_call(self) -> bool:
670 + return self._needs_tool_call
671 +
672 + async def call_tool(self):
673 + self._needs_tool_call = False
674 + return []
675 +
676 + def append_tool_output(self, output) -> None:
677 + self._messages.extend(output)
678 +
679 + async def generate_output():
680 + yield MagicMock()
681 +
682 + context = ToolCallingHarmonyContext()
683 + serving_responses_instance.engine_client.generate.side_effect = (
684 + lambda *args, **kwargs: generate_output()
685 + )
686 + monkeypatch.setattr(
687 + "vllm.renderers.online_renderer.render_for_completion",
688 + lambda messages: [1, 2, 3, 4],
689 + )
690 + online_renderer = serving_responses_instance.online_renderer
691 + online_renderer.renderer = SimpleNamespace(
692 + tokenizer=SimpleNamespace(truncation_side="left")
693 + )
694 + serving_responses_instance.online_renderer.render_responses_harmony_messages = (
695 + OnlineRenderer.render_responses_harmony_messages.__get__(online_renderer)
696 + )
697 + serving_responses_instance._extract_prompt_len = MagicMock(return_value=2)
698 +
699 + async for _ in serving_responses_instance._generate_with_builtin_tools(
700 + request_id="req",
701 + engine_input=tokens_input([1], cache_salt="request-salt"),
702 + sampling_params=MagicMock(),
703 + context=context,
704 + tok_params=TokenizeParams(
705 + max_total_tokens=3,
706 + max_output_tokens=1,
707 + truncate_prompt_tokens=-1,
708 + ),
709 + ):
710 + pass
711 +
712 + assert serving_responses_instance.engine_client.generate.call_count == 2
713 + followup_engine_input = (
714 + serving_responses_instance.engine_client.generate.call_args_list[1].args[0]
715 + )
716 + assert followup_engine_input.get("cache_salt") == "request-salt"
717 + assert followup_engine_input["prompt_token_ids"] == [3, 4]
718 +
719 @pytest.mark.asyncio
720 async def test_initialize_tool_sessions(
721 self, serving_responses_instance, mock_context, mock_exit_stack
tests/entrypoints/openai/test_render_parity.py
+2 -1
@@ -151,7 +151,8 @@ async def _capture_responses(
151 request: ResponsesRequest,
152 ) -> CapturedRenderInputs:
153 capture = RenderCapture(serving.online_renderer)
154 - await serving._make_request(request, prev_response=None)
154 + result = await serving.online_renderer.render_responses(request)
155 + assert not isinstance(result, ErrorResponse), result
156 return capture.take()
157
158
tests/entrypoints/scale_out/render/test_render.py
+188
@@ -3,15 +3,138 @@
3
4 """Tests for the /render endpoints that expose prompt preprocessing."""
5
6 +from http import HTTPStatus
7 +from types import SimpleNamespace
8 +from unittest.mock import AsyncMock, MagicMock
9 +
10 import httpx
11 import pytest
12 import pytest_asyncio
13
14 from tests.utils import RemoteLaunchRenderServer
15 +from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
16 +from vllm.entrypoints.scale_out.render.api_router import router
17 +from vllm.entrypoints.scale_out.render.serving import ServingRender
18 +from vllm.entrypoints.serve.engine.protocol import ErrorResponse
19 +from vllm.renderers.online_renderer import OnlineRenderer
20
21 MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
22
23
24 +def _build_responses_serving_render() -> ServingRender:
25 + serving = ServingRender.__new__(ServingRender)
26 + serving.model_config = SimpleNamespace(
27 + max_model_len=100,
28 + is_encoder_decoder=False,
29 + )
30 + serving.default_sampling_params = {}
31 + serving.override_max_tokens = None
32 + serving.tool_server = MagicMock()
33 + serving.online_renderer = MagicMock()
34 + serving.online_renderer.create_error_response = (
35 + OnlineRenderer.create_error_response.__get__(serving.online_renderer)
36 + )
37 + serving.online_renderer.validate_chat_template = (
38 + OnlineRenderer.validate_chat_template.__get__(serving.online_renderer)
39 + )
40 + serving.online_renderer.trust_request_chat_template = True
41 + serving._check_model = AsyncMock(return_value=None)
42 + return serving
43 +
44 +
45 +@pytest.mark.skip_global_cleanup
46 +def test_responses_render_route_is_registered():
47 + assert any(route.path == "/v1/responses/render" for route in router.routes)
48 +
49 +
50 +@pytest.mark.skip_global_cleanup
51 +def test_responses_render_route_documents_not_implemented():
52 + route = next(
53 + route for route in router.routes if route.path == "/v1/responses/render"
54 + )
55 +
56 + assert HTTPStatus.NOT_IMPLEMENTED.value in route.responses
57 +
58 +
59 +@pytest.mark.asyncio
60 +@pytest.mark.skip_global_cleanup
61 +async def test_render_responses_returns_generate_request_without_stored_state():
62 + serving = _build_responses_serving_render()
63 + serving.online_renderer.render_responses = AsyncMock(
64 + return_value=MagicMock(
65 + messages=[{"role": "user", "content": "Test prompt"}],
66 + engine_input={"prompt_token_ids": [7, 8, 9]},
67 + )
68 + )
69 + request = ResponsesRequest(
70 + model=MODEL_NAME,
71 + input="Test prompt",
72 + max_output_tokens=12,
73 + stream=True,
74 + cache_salt="request-salt",
75 + priority=3,
76 + kv_transfer_params={"do_remote_prefill": True},
77 + ec_transfer_params={"remote": True},
78 + )
79 +
80 + response = await serving.render_responses_request(request)
81 +
82 + assert response.token_ids == [7, 8, 9]
83 + assert response.request_id == request.request_id
84 + assert response.sampling_params.max_tokens == 12
85 + assert response.model == MODEL_NAME
86 + assert response.stream is True
87 + assert response.cache_salt == "request-salt"
88 + assert response.priority == 3
89 + assert response.kv_transfer_params == {"do_remote_prefill": True}
90 + assert response.ec_transfer_params == {"remote": True}
91 + serving.online_renderer.render_responses.assert_awaited_once_with(
92 + request,
93 + previous_messages=None,
94 + previous_response_outputs=None,
95 + tool_server=serving.tool_server,
96 + skip_mm_cache=True,
97 + )
98 +
99 +
100 +@pytest.mark.asyncio
101 +@pytest.mark.skip_global_cleanup
102 +async def test_render_responses_rejects_previous_response_id():
103 + serving = _build_responses_serving_render()
104 + serving.online_renderer.render_responses = AsyncMock()
105 + request = ResponsesRequest(
106 + model=MODEL_NAME,
107 + input="Test prompt",
108 + previous_response_id="resp_previous",
109 + )
110 +
111 + response = await serving.render_responses_request(request)
112 +
113 + assert isinstance(response, ErrorResponse)
114 + assert response.error.code == 400
115 + assert response.error.param == "previous_response_id"
116 + serving.online_renderer.render_responses.assert_not_awaited()
117 +
118 +
119 +@pytest.mark.asyncio
120 +@pytest.mark.skip_global_cleanup
121 +async def test_render_responses_rejects_empty_token_ids():
122 + serving = _build_responses_serving_render()
123 + serving.online_renderer.render_responses = AsyncMock(
124 + return_value=MagicMock(
125 + messages=[],
126 + engine_input={"prompt_token_ids": []},
127 + )
128 + )
129 +
130 + response = await serving.render_responses_request(
131 + ResponsesRequest(model=MODEL_NAME, input="Test prompt")
132 + )
133 +
134 + assert isinstance(response, ErrorResponse)
135 + assert response.error.message == "No token_ids rendered"
136 +
137 +
138 @pytest.fixture(scope="module")
139 def server():
140 args: list[str] = ["--trust-request-chat-template"]
@@ -28,6 +151,71 @@ async def client(server):
151 yield http_client
152
153
154 +@pytest.mark.asyncio
155 +async def test_responses_render_basic(client):
156 + response = await client.post(
157 + "/v1/responses/render",
158 + json={
159 + "model": MODEL_NAME,
160 + "input": "When should a Responses handler return an empty string?",
161 + "max_output_tokens": 7,
162 + },
163 + )
164 +
165 + assert response.status_code == 200
166 + data = response.json()
167 + assert data["model"] == MODEL_NAME
168 + assert data["request_id"].startswith("resp_")
169 + assert data["sampling_params"]["max_tokens"] == 7
170 + assert data["token_ids"]
171 +
172 +
173 +@pytest.mark.asyncio
174 +async def test_responses_render_includes_prior_input_items(client):
175 + current_turn = {
176 + "role": "user",
177 + "content": "Which color should I remember?",
178 + }
179 + current_turn_response = await client.post(
180 + "/v1/responses/render",
181 + json={"model": MODEL_NAME, "input": [current_turn]},
182 + )
183 + multi_turn_response = await client.post(
184 + "/v1/responses/render",
185 + json={
186 + "model": MODEL_NAME,
187 + "input": [
188 + {"role": "user", "content": "Remember the color cobalt."},
189 + {"role": "assistant", "content": "I will remember cobalt."},
190 + current_turn,
191 + ],
192 + },
193 + )
194 +
195 + assert current_turn_response.status_code == 200
196 + assert multi_turn_response.status_code == 200
197 + current_turn_token_ids = current_turn_response.json()["token_ids"]
198 + multi_turn_token_ids = multi_turn_response.json()["token_ids"]
199 + assert len(multi_turn_token_ids) > len(current_turn_token_ids)
200 +
201 +
202 +@pytest.mark.asyncio
203 +async def test_responses_render_rejects_previous_response_id_over_http(client):
204 + response = await client.post(
205 + "/v1/responses/render",
206 + json={
207 + "model": MODEL_NAME,
208 + "input": "Continue the previous response.",
209 + "previous_response_id": "resp_previous",
210 + },
211 + )
212 +
213 + assert response.status_code == 400
214 + error = response.json()["error"]
215 + assert error["type"] == "invalid_request_error"
216 + assert error["param"] == "previous_response_id"
217 +
218 +
219 @pytest.mark.asyncio
220 async def test_completion_render_basic(client):
221 """Test basic completion render endpoint."""
tests/entrypoints/scale_out/test_factories.py
+1
@@ -15,6 +15,7 @@ RENDER_PATHS = {
15 "/v1/completions/render",
16 "/v1/completions/derender",
17 "/v1/messages/render",
18 + "/v1/responses/render",
19 }
20 GENERATE_PATH = "/inference/v1/generate"
21 SCALE_OUT_PATHS = RENDER_PATHS | {GENERATE_PATH}
vllm/entrypoints/generate/api_router.py
+6 -33
@@ -1,6 +1,6 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -from typing import TYPE_CHECKING
3 +from typing import TYPE_CHECKING, Any
4
5 from fastapi import FastAPI
6
@@ -60,6 +60,7 @@ async def init_generate_state(
60 args: "Namespace",
61 request_logger: RequestLogger | None,
62 supported_tasks: tuple["SupportedTask", ...],
63 + default_chat_template_kwargs: dict[str, Any],
64 ):
65 from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
66 from vllm.entrypoints.chat_utils import load_chat_template
@@ -79,12 +80,6 @@ async def init_generate_state(
80 CohereServingChatV2 = None # type: ignore[assignment,misc]
81 else:
82 CohereServingChatV2 = None # type: ignore[assignment,misc]
82 -
83 - from vllm.entrypoints.mcp.tool_server import (
84 - DemoToolServer,
85 - MCPToolServer,
86 - ToolServer,
87 - )
83 from vllm.entrypoints.openai.chat_completion.batch_serving import (
84 OpenAIServingChatBatch,
85 )
@@ -100,33 +95,11 @@ async def init_generate_state(
95 getattr(args, "fingerprint_value", None),
96 )
97
103 - if args.tool_server == "demo":
104 - tool_server: ToolServer | None = DemoToolServer()
105 - assert isinstance(tool_server, DemoToolServer)
106 - await tool_server.init_and_validate()
107 - elif args.tool_server:
108 - tool_server = MCPToolServer()
109 - await tool_server.add_tool_server(args.tool_server)
110 - else:
111 - tool_server = None
98 resolved_chat_template = load_chat_template(args.chat_template)
99
114 - # Fold the dedicated ``--cohere-format`` CLI flag into the renderer's
115 - # default chat-template kwargs. The cohere renderer reads
116 - # ``chat_template_kwargs["cohere_format"]`` to pick cmd3 vs cmd4
117 - # rendering; making this a first-class flag keeps the right format
118 - # discoverable for ``vllm serve --tokenizer-mode cohere`` users
119 - # without forcing them to hand-construct a JSON dict for
120 - # ``--default-chat-template-kwargs``. Per-request overrides still
121 - # take precedence (see ``merge_kwargs`` in
122 - # ``ChatCompletionRequest.build_chat_params``).
123 - default_chat_template_kwargs = dict(args.default_chat_template_kwargs or {})
124 - if getattr(args, "cohere_format", None):
125 - default_chat_template_kwargs.setdefault("cohere_format", args.cohere_format)
126 -
127 - # Render endpoints are always backed by OnlineRenderer so that
128 - # /v1/chat/completions/render and /v1/completions/render work on both
129 - # generate-mode and render-only servers. Created in init_app_state.
100 + # Render endpoints are always backed by OnlineRenderer so that chat,
101 + # completion, and Responses rendering work on both generate-mode and
102 + # render-only servers. Created in init_app_state.
103
104 state.openai_serving_responses = (
105 OpenAIServingResponses(
@@ -139,7 +112,7 @@ async def init_generate_state(
112 return_tokens_as_token_ids=args.return_tokens_as_token_ids,
113 enable_auto_tools=args.enable_auto_tool_choice,
114 tool_parser=args.tool_call_parser,
142 - tool_server=tool_server,
115 + tool_server=state.tool_server,
116 reasoning_parser=args.structured_outputs_config.reasoning_parser,
117 enable_prompt_tokens_details=args.enable_prompt_tokens_details,
118 enable_force_include_usage=args.enable_force_include_usage,
vllm/entrypoints/launchers/api_server/app_state.py
+15 -4
@@ -8,6 +8,8 @@ from starlette.datastructures import State
8
9 from vllm.engine.protocol import EngineClient
10 from vllm.entrypoints.chat_utils import load_chat_template
11 +from vllm.entrypoints.launchers.cli_args import resolve_default_chat_template_kwargs
12 +from vllm.entrypoints.mcp.tool_server import init_tool_server
13 from vllm.entrypoints.openai.models.protocol import BaseModelPath
14 from vllm.entrypoints.openai.models.serving import OpenAIServingModels
15 from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
@@ -63,6 +65,10 @@ async def init_app_state(
65 state.vllm_config = vllm_config
66 state.args = args
67 resolved_chat_template = load_chat_template(args.chat_template)
68 + default_chat_template_kwargs = resolve_default_chat_template_kwargs(args)
69 + state.tool_server = (
70 + await init_tool_server(args) if "generate" in supported_tasks else None
71 + )
72
73 # Merge default_mm_loras into the static lora_modules
74 default_mm_loras = (
@@ -90,7 +96,7 @@ async def init_app_state(
96 exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
97 tool_parser=args.tool_call_parser,
98 reasoning_parser=args.structured_outputs_config.reasoning_parser,
93 - default_chat_template_kwargs=args.default_chat_template_kwargs,
99 + default_chat_template_kwargs=default_chat_template_kwargs,
100 log_error_stack=args.log_error_stack,
101 )
102 state.online_renderer.warmup()
@@ -106,7 +112,7 @@ async def init_app_state(
112 exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
113 tool_parser=args.tool_call_parser,
114 reasoning_parser=args.structured_outputs_config.reasoning_parser,
109 - default_chat_template_kwargs=args.default_chat_template_kwargs,
115 + default_chat_template_kwargs=default_chat_template_kwargs,
116 log_error_stack=args.log_error_stack,
117 )
118
@@ -116,7 +122,7 @@ async def init_app_state(
122 request_logger=request_logger,
123 chat_template=resolved_chat_template,
124 chat_template_content_format=args.chat_template_content_format,
119 - default_chat_template_kwargs=args.default_chat_template_kwargs,
125 + default_chat_template_kwargs=default_chat_template_kwargs,
126 trust_request_chat_template=args.trust_request_chat_template,
127 )
128
@@ -124,7 +130,12 @@ async def init_app_state(
130 from vllm.entrypoints.generate.api_router import init_generate_state
131
132 await init_generate_state(
127 - engine_client, state, args, request_logger, supported_tasks
133 + engine_client,
134 + state,
135 + args,
136 + request_logger,
137 + supported_tasks,
138 + default_chat_template_kwargs,
139 )
140
141 from vllm.entrypoints.scale_out.factories import init_scale_out_state
vllm/entrypoints/launchers/cli_args.py
+10
@@ -370,6 +370,16 @@ class FrontendArgs(BaseFrontendArgs):
370 return frontend_kwargs
371
372
373 +def resolve_default_chat_template_kwargs(
374 + args: argparse.Namespace,
375 +) -> dict[str, Any]:
376 + """Resolve renderer defaults, including the dedicated Cohere format flag."""
377 + kwargs = dict(args.default_chat_template_kwargs or {})
378 + if getattr(args, "cohere_format", None):
379 + kwargs.setdefault("cohere_format", args.cohere_format)
380 + return kwargs
381 +
382 +
383 def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
384 """Create the CLI argument parser used by the OpenAI API server.
385
vllm/entrypoints/launchers/render/app_state.py
+7 -3
@@ -6,6 +6,8 @@ from starlette.datastructures import State
6
7 from vllm.config import VllmConfig
8 from vllm.entrypoints.chat_utils import load_chat_template
9 +from vllm.entrypoints.launchers.cli_args import resolve_default_chat_template_kwargs
10 +from vllm.entrypoints.mcp.tool_server import init_tool_server
11 from vllm.entrypoints.openai.models.protocol import BaseModelPath
12 from vllm.entrypoints.openai.models.serving import OpenAIModelRegistry
13 from vllm.entrypoints.scale_out.factories import init_render_state
@@ -46,6 +48,8 @@ async def init_render_app_state(
48
49 renderer = renderer_from_config(vllm_config)
50 resolved_chat_template = load_chat_template(args.chat_template)
51 + default_chat_template_kwargs = resolve_default_chat_template_kwargs(args)
52 + state.tool_server = await init_tool_server(args)
53
54 state.online_renderer = OnlineRenderer(
55 model_config=vllm_config.model_config,
@@ -58,7 +62,7 @@ async def init_render_app_state(
62 exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
63 tool_parser=args.tool_call_parser,
64 reasoning_parser=args.reasoning_parser,
61 - default_chat_template_kwargs=args.default_chat_template_kwargs,
65 + default_chat_template_kwargs=default_chat_template_kwargs,
66 log_error_stack=args.log_error_stack,
67 )
68 state.online_renderer.warmup()
@@ -74,7 +78,7 @@ async def init_render_app_state(
78 exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
79 tool_parser=args.tool_call_parser,
80 reasoning_parser=args.reasoning_parser,
77 - default_chat_template_kwargs=args.default_chat_template_kwargs,
81 + default_chat_template_kwargs=default_chat_template_kwargs,
82 log_error_stack=args.log_error_stack,
83 )
84
@@ -85,7 +89,7 @@ async def init_render_app_state(
89 request_logger=request_logger,
90 chat_template=resolved_chat_template,
91 chat_template_content_format=args.chat_template_content_format,
88 - default_chat_template_kwargs=args.default_chat_template_kwargs,
92 + default_chat_template_kwargs=default_chat_template_kwargs,
93 trust_request_chat_template=args.trust_request_chat_template,
94 )
95
vllm/entrypoints/mcp/tool_server.py
+15
@@ -12,6 +12,8 @@ from vllm.logger import init_logger
12 logger = init_logger(__name__)
13
14 if TYPE_CHECKING:
15 + from argparse import Namespace
16 +
17 from mcp.types import ListToolsResult
18
19
@@ -232,3 +234,16 @@ class DemoToolServer(ToolServer):
234 if tool_name not in self.tools:
235 raise KeyError(f"Tool '{tool_name}' is not supported")
236 yield self.tools[tool_name]
237 +
238 +
239 +async def init_tool_server(args: "Namespace") -> ToolServer | None:
240 + tool_server = getattr(args, "tool_server", None)
241 + if tool_server == "demo":
242 + demo_server = DemoToolServer()
243 + await demo_server.init_and_validate()
244 + return demo_server
245 + if tool_server:
246 + mcp_server = MCPToolServer()
247 + await mcp_server.add_tool_server(tool_server)
248 + return mcp_server
249 + return None
vllm/entrypoints/openai/responses/serving.py
+76 -252
@@ -6,13 +6,11 @@ import time
6 from collections import deque
7 from collections.abc import AsyncGenerator, AsyncIterator, Callable, Mapping, Sequence
8 from contextlib import AsyncExitStack
9 -from copy import copy
9 from http import HTTPStatus
11 -from typing import Any, Final
10 +from typing import Any, Final, cast
11
12 from fastapi import Request
13 from openai.types.responses import (
15 - ResponseFunctionToolCall,
14 ResponseOutputItem,
15 ResponseOutputMessage,
16 ResponseOutputText,
@@ -20,15 +18,12 @@ from openai.types.responses import (
18 response_text_delta_event,
19 )
20 from openai.types.responses.response_output_text import Logprob, LogprobTopLogprob
23 -from openai.types.responses.tool import Mcp, Tool
24 -from openai_harmony import Message as OpenAIHarmonyMessage
21 from pydantic import TypeAdapter
22
23 from vllm import envs
24 from vllm.config.utils import replace
25 from vllm.engine.protocol import EngineClient
26 from vllm.entrypoints.chat_utils import (
31 - ChatCompletionMessageParam,
27 ChatTemplateContentFormatOption,
28 )
29 from vllm.entrypoints.generate.base.protocol import (
@@ -38,24 +33,13 @@ from vllm.entrypoints.generate.base.protocol import (
33 from vllm.entrypoints.generate.base.serving import GenerateBaseServing
34 from vllm.entrypoints.mcp.tool_server import ToolServer
35 from vllm.entrypoints.openai.models.serving import OpenAIServingModels
41 -from vllm.entrypoints.openai.parser.harmony_utils import (
42 - build_harmony_preamble,
43 - extract_instructions_from_messages,
44 - get_user_message,
45 - has_custom_tools,
46 - render_for_completion,
47 -)
36 from vllm.entrypoints.openai.responses.context import (
37 ConversationContext,
38 HarmonyContext,
39 ParsableContext,
40 SimpleContext,
41 )
54 -from vllm.entrypoints.openai.responses.harmony import (
55 - construct_harmony_previous_input_messages,
56 - harmony_to_response_output,
57 - response_input_to_harmony,
58 -)
42 +from vllm.entrypoints.openai.responses.harmony import harmony_to_response_output
43 from vllm.entrypoints.openai.responses.protocol import (
44 InputTokensDetails,
45 OutputTokensDetails,
@@ -80,8 +64,6 @@ from vllm.entrypoints.openai.responses.streaming_events import (
64 )
65 from vllm.entrypoints.openai.responses.utils import (
66 build_response_output_items,
83 - construct_input_messages,
84 - construct_tool_dicts,
67 extract_function_tool_names,
68 extract_tool_types,
69 )
@@ -89,14 +71,19 @@ from vllm.entrypoints.serve.engine.protocol import ErrorResponse
71 from vllm.entrypoints.serve.utils.api_utils import get_max_tokens
72 from vllm.entrypoints.serve.utils.request_logger import RequestLogger
73 from vllm.exceptions import GenerationError, VLLMValidationError
92 -from vllm.inputs import EngineInput, tokens_input
74 +from vllm.inputs import EngineInput
75 from vllm.logger import init_logger
76 from vllm.logprobs import Logprob as SampleLogprob
77 from vllm.logprobs import SampleLogprobs
78 from vllm.lora.request import LoRARequest
79 from vllm.outputs import CompletionOutput
80 from vllm.parser import Parser, ParserManager
99 -from vllm.renderers.online_renderer import OnlineRenderer
81 +from vllm.renderers import TokenizeParams
82 +from vllm.renderers.online_renderer import (
83 + OnlineRenderer,
84 + ResponsesPreviousMessages,
85 + ResponsesRenderResult,
86 +)
87 from vllm.sampling_params import SamplingParams, StructuredOutputsParams
88 from vllm.tokenizers import TokenizerLike
89 from vllm.utils import random_uuid
@@ -105,45 +92,6 @@ from vllm.utils.collection_utils import as_list
92 logger = init_logger(__name__)
93
94
108 -def _extract_allowed_tools_from_mcp_requests(
109 - tools: list[Tool],
110 -) -> dict[str, list[str] | None]:
111 - """
112 - Extract allowed_tools mapping from MCP tool requests.
113 -
114 - Returns a dictionary mapping server_label to allowed_tools list.
115 - Handles both list format and McpAllowedToolsMcpToolFilter object format.
116 -
117 - Special handling:
118 - - If allowed_tools is None, returns None (allows all tools)
119 - - If allowed_tools contains "*", returns None (allows all tools)
120 - - Otherwise, returns the list of specific tool names
121 -
122 - This function can be reused for both harmony and non-harmony MCP calls.
123 - """
124 - allowed_tools_map: dict[str, list[str] | None] = {}
125 - for tool in tools:
126 - if not isinstance(tool, Mcp):
127 - continue
128 -
129 - # allowed_tools can be a list or an object with tool_names
130 - # Extract the actual list of tool names
131 - allowed_tools_val = None
132 - if tool.allowed_tools is not None:
133 - if isinstance(tool.allowed_tools, list):
134 - allowed_tools_val = tool.allowed_tools
135 - elif hasattr(tool.allowed_tools, "tool_names"):
136 - # It's an McpAllowedToolsMcpToolFilter object
137 - allowed_tools_val = tool.allowed_tools.tool_names
138 -
139 - # Normalize "*" to None (both mean "allow all tools")
140 - if allowed_tools_val is not None and "*" in allowed_tools_val:
141 - allowed_tools_val = None
142 -
143 - allowed_tools_map[tool.server_label] = allowed_tools_val
144 - return allowed_tools_map
145 -
146 -
95 class OpenAIServingResponses(GenerateBaseServing):
96 def __init__(
97 self,
@@ -226,7 +174,7 @@ class OpenAIServingResponses(GenerateBaseServing):
174 # HACK(woosuk): This is a hack. We should use a better store.
175 # FIXME: If enable_store=True, this may cause a memory leak since we
176 # never remove messages from the store.
229 - self.msg_store: dict[str, list[ChatCompletionMessageParam]] = {}
177 + self.msg_store: dict[str, ResponsesPreviousMessages] = {}
178
179 # HACK(wuhang): This is a hack. We should use a better store.
180 # FIXME: If enable_store=True, this may cause a memory leak since we
@@ -322,6 +270,33 @@ class OpenAIServingResponses(GenerateBaseServing):
270 )
271 return None
272
273 + async def _render_resolved_response_inputs(
274 + self,
275 + request: ResponsesRequest,
276 + prev_response: ResponsesResponse | None,
277 + *,
278 + skip_mm_cache: bool = False,
279 + ) -> ResponsesRenderResult | ErrorResponse:
280 + previous_messages: ResponsesPreviousMessages | None = None
281 + previous_response_outputs: list[ResponseOutputItem] | None = None
282 + if prev_response is not None:
283 + stored_messages = (
284 + self.msg_store[prev_response.id]
285 + if self.use_harmony
286 + else self.msg_store.get(prev_response.id)
287 + )
288 + if stored_messages is not None:
289 + previous_messages = stored_messages.copy()
290 + previous_response_outputs = list(prev_response.output)
291 +
292 + return await self.online_renderer.render_responses(
293 + request,
294 + previous_messages=previous_messages,
295 + previous_response_outputs=previous_response_outputs,
296 + tool_server=self.tool_server,
297 + skip_mm_cache=skip_mm_cache,
298 + )
299 +
300 async def create_responses(
301 self,
302 request: ResponsesRequest,
@@ -374,12 +349,14 @@ class OpenAIServingResponses(GenerateBaseServing):
349 lora_request = self._maybe_get_adapters(request)
350 model_name = self.models.model_name(lora_request)
351
377 - if self.use_harmony:
378 - messages, engine_inputs = self._make_request_with_harmony(
379 - request, prev_response
380 - )
381 - else:
382 - messages, engine_inputs = await self._make_request(request, prev_response)
352 + render_result = await self._render_resolved_response_inputs(
353 + request,
354 + prev_response,
355 + )
356 + if isinstance(render_result, ErrorResponse):
357 + return render_result
358 + messages = render_result.messages
359 + engine_inputs = [render_result.engine_input]
360
361 request_metadata = RequestResponseMetadata(request_id=request.request_id)
362 if raw_request:
@@ -508,6 +485,11 @@ class OpenAIServingResponses(GenerateBaseServing):
485 engine_input=engine_input,
486 sampling_params=sampling_params,
487 context=context,
488 + tok_params=(
489 + request.build_tok_params(self.model_config)
490 + if isinstance(context, HarmonyContext)
491 + else None
492 + ),
493 lora_request=lora_request,
494 priority=self._get_priority(request, raw_request),
495 trace_headers=trace_headers,
@@ -599,60 +581,28 @@ class OpenAIServingResponses(GenerateBaseServing):
581 request_metadata,
582 )
583
602 - async def _make_request(
603 - self,
604 - request: ResponsesRequest,
605 - prev_response: ResponsesResponse | None,
606 - ):
607 - tool_dicts = construct_tool_dicts(
608 - request.tools,
609 - request.tool_choice,
610 - exclude_tools_when_tool_choice_none=(
611 - self.online_renderer.exclude_tools_when_tool_choice_none
612 - ),
613 - )
614 - # Construct the input messages.
615 - messages = construct_input_messages(
616 - request_instructions=request.instructions,
617 - request_input=request.input,
618 - prev_msg=self.msg_store.get(prev_response.id) if prev_response else None,
619 - prev_response_output=prev_response.output if prev_response else None,
620 - )
621 - chat_template_kwargs = self._effective_chat_template_kwargs(request)
622 - _, engine_inputs = await self.online_renderer.preprocess_chat(
623 - request,
624 - messages,
625 - default_template=self.chat_template,
626 - default_template_content_format=self.chat_template_content_format,
627 - default_template_kwargs=chat_template_kwargs,
628 - tool_dicts=tool_dicts,
629 - parser=self.parser,
630 - )
631 - return messages, engine_inputs
632 -
584 async def _render_next_turn(
585 self,
586 request: ResponsesRequest,
587 messages: list[ResponseInputOutputItem],
637 - tool_dicts: list[dict[str, Any]] | None,
638 - parser: type[Parser] | None,
639 - chat_template: str | None,
640 - chat_template_content_format: ChatTemplateContentFormatOption,
641 - ):
642 - new_messages = construct_input_messages(
643 - request_input=messages,
588 + ) -> list[EngineInput]:
589 + render_request = request.model_copy(
590 + update={
591 + "input": list(messages),
592 + "instructions": None,
593 + "previous_input_messages": None,
594 + "previous_response_id": None,
595 + }
596 )
645 - chat_template_kwargs = self._effective_chat_template_kwargs(request)
646 - _, engine_inputs = await self.online_renderer.preprocess_chat(
647 - request,
648 - new_messages,
649 - default_template=chat_template,
650 - default_template_content_format=chat_template_content_format,
651 - default_template_kwargs=chat_template_kwargs,
652 - tool_dicts=tool_dicts,
653 - parser=parser,
597 + render_result = await self.online_renderer.render_responses(
598 + render_request,
599 + previous_messages=None,
600 + previous_response_outputs=None,
601 + tool_server=self.tool_server,
602 )
655 - return engine_inputs
603 + if isinstance(render_result, ErrorResponse):
604 + raise VLLMValidationError(render_result.error.message)
605 + return [render_result.engine_input]
606
607 async def _generate_with_builtin_tools(
608 self,
@@ -660,6 +610,7 @@ class OpenAIServingResponses(GenerateBaseServing):
610 engine_input: EngineInput,
611 sampling_params: SamplingParams,
612 context: ConversationContext,
613 + tok_params: TokenizeParams | None = None,
614 lora_request: LoRARequest | None = None,
615 priority: int = 0,
616 trace_headers: Mapping[str, str] | None = None,
@@ -667,6 +618,7 @@ class OpenAIServingResponses(GenerateBaseServing):
618 reasoning_parser_kwargs: dict[str, Any] | None = None,
619 ):
620 max_model_len = self.model_config.max_model_len
621 + cache_salt = cast(str | None, engine_input.get("cache_salt"))
622
623 orig_priority = priority
624 sub_request = 0
@@ -711,18 +663,19 @@ class OpenAIServingResponses(GenerateBaseServing):
663 # Create inputs for the next turn.
664 # Render the next prompt token ids and update sampling_params.
665 if isinstance(context, HarmonyContext):
714 - token_ids = context.render_for_completion()
715 - engine_input = tokens_input(token_ids)
666 + engine_input = self.online_renderer.render_responses_harmony_messages(
667 + context.messages,
668 + cache_salt=cache_salt,
669 + tok_params=tok_params,
670 + )
671
717 - sampling_params.max_tokens = max_model_len - len(token_ids)
672 + sampling_params.max_tokens = max_model_len - self._extract_prompt_len(
673 + engine_input
674 + )
675 elif isinstance(context, ParsableContext):
676 (engine_input,) = await self._render_next_turn(
677 context.request,
678 context.response_messages,
722 - context.tool_dicts,
723 - context.parser_cls,
724 - context.chat_template,
725 - context.chat_template_content_format,
679 )
680
681 sampling_params.max_tokens = get_max_tokens(
@@ -740,28 +693,6 @@ class OpenAIServingResponses(GenerateBaseServing):
693 priority = orig_priority - 1
694 sub_request += 1
695
743 - def _make_request_with_harmony(
744 - self,
745 - request: ResponsesRequest,
746 - prev_response: ResponsesResponse | None,
747 - ):
748 - if self.parser is not None:
749 - # HarmonyParser doesn't need chat_template_kwargs
750 - # TODO: Unify adjust_request() call with non-harmony branch
751 - self.parser(
752 - self.renderer.get_tokenizer(),
753 - request.tools,
754 - model_config=self.model_config,
755 - ).adjust_request(request=request)
756 -
757 - arrival_time = time.time()
758 - messages = self._construct_input_messages_with_harmony(request, prev_response)
759 - prompt_token_ids = render_for_completion(messages)
760 - engine_input = tokens_input(prompt_token_ids, cache_salt=request.cache_salt)
761 - engine_input["arrival_time"] = arrival_time
762 -
763 - return messages, [engine_input]
764 -
696 async def _initialize_tool_sessions(
697 self,
698 request: ResponsesRequest,
@@ -1104,113 +1035,6 @@ class OpenAIServingResponses(GenerateBaseServing):
1035 )
1036 ]
1037
1107 - def _get_harmony_builtin_tool_descriptions(
1108 - self, request: ResponsesRequest, tool_types: set[str]
1109 - ) -> dict[str, str | None]:
1110 - # Extract allowed_tools from MCP tool requests
1111 - allowed_tools_map = _extract_allowed_tools_from_mcp_requests(request.tools)
1112 -
1113 - # Get filtered tool descriptions first.
1114 - # If get_tool_description returns None (due to filtering), the tool is disabled.
1115 - browser_description = (
1116 - self.tool_server.get_tool_description(
1117 - "browser", allowed_tools_map.get("web_search_preview")
1118 - )
1119 - if "web_search_preview" in tool_types
1120 - and self.tool_server is not None
1121 - and self.tool_server.has_tool("browser")
1122 - else None
1123 - )
1124 - python_description = (
1125 - self.tool_server.get_tool_description(
1126 - "python", allowed_tools_map.get("code_interpreter")
1127 - )
1128 - if "code_interpreter" in tool_types
1129 - and self.tool_server is not None
1130 - and self.tool_server.has_tool("python")
1131 - else None
1132 - )
1133 - container_description = (
1134 - self.tool_server.get_tool_description(
1135 - "container", allowed_tools_map.get("container")
1136 - )
1137 - if "container" in tool_types
1138 - and self.tool_server is not None
1139 - and self.tool_server.has_tool("container")
1140 - else None
1141 - )
1142 - return {
1143 - "browser_description": browser_description,
1144 - "python_description": python_description,
1145 - "container_description": container_description,
1146 - }
1147 -
1148 - def _construct_input_messages_with_harmony(
1149 - self,
1150 - request: ResponsesRequest,
1151 - prev_response: ResponsesResponse | None,
1152 - ) -> list[OpenAIHarmonyMessage]:
1153 - messages: list[OpenAIHarmonyMessage] = []
1154 - request_input = request.input
1155 - if prev_response is None:
1156 - # New conversation.
1157 - tool_types = extract_tool_types(request.tools)
1158 - with_custom_tools = has_custom_tools(tool_types)
1159 - instructions = request.instructions
1160 - if instructions is None and isinstance(request_input, list):
1161 - instructions, request_input = extract_instructions_from_messages(
1162 - request_input
1163 - )
1164 - tool_descriptions = self._get_harmony_builtin_tool_descriptions(
1165 - request, tool_types
1166 - )
1167 - tools = request.tools if with_custom_tools else None
1168 - messages.extend(
1169 - build_harmony_preamble(
1170 - instructions=instructions,
1171 - tools=tools,
1172 - reasoning_effort=(
1173 - request.reasoning.effort if request.reasoning else None
1174 - ),
1175 - with_custom_tools=with_custom_tools,
1176 - **tool_descriptions,
1177 - )
1178 - )
1179 - messages += construct_harmony_previous_input_messages(request)
1180 -
1181 - else:
1182 - # Continue the previous conversation.
1183 - # FIXME(woosuk): Currently, request params like reasoning and
1184 - # instructions are ignored.
1185 - prev_msgs = self.msg_store[prev_response.id]
1186 -
1187 - messages.extend(prev_msgs)
1188 - # Append the new input.
1189 - # Responses API supports simple text inputs without chat format.
1190 - if isinstance(request_input, str):
1191 - # Skip empty string input when previous_input_messages supplies
1192 - # the full conversation history --- an empty trailing user message
1193 - # confuses the model into thinking nothing was sent.
1194 - if request_input or not request.previous_input_messages:
1195 - messages.append(get_user_message(request_input))
1196 - else:
1197 - if prev_response is not None:
1198 - prev_outputs = copy(prev_response.output)
1199 - else:
1200 - prev_outputs = []
1201 - for response_msg in request_input:
1202 - new_msg = response_input_to_harmony(response_msg, prev_outputs)
1203 - if new_msg is not None:
1204 - messages.append(new_msg)
1205 -
1206 - # User passes in a tool call request and its output. We need
1207 - # to add the tool call request to prev_outputs so that
1208 - # response_input_to_harmony can find the tool call request when
1209 - # parsing the tool call output.
1210 - if isinstance(response_msg, ResponseFunctionToolCall):
1211 - prev_outputs.append(response_msg)
1212 - return messages
1213 -
1038 async def _run_background_request_stream(
1039 self,
1040 request: ResponsesRequest,
vllm/entrypoints/scale_out/factories.py
+1
@@ -31,6 +31,7 @@ def init_render_state(
31 state.openai_serving_models,
32 state.online_renderer,
33 request_logger=request_logger,
34 + tool_server=state.tool_server,
35 )
36
37 state.serving_derender = ServingDerender(
vllm/entrypoints/scale_out/render/api_router.py
+23
@@ -8,6 +8,7 @@ from fastapi.responses import JSONResponse
8 from vllm.entrypoints.anthropic.protocol import AnthropicMessagesRequest
9 from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
10 from vllm.entrypoints.openai.completion.protocol import CompletionRequest
11 +from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
12 from vllm.entrypoints.serve.engine.protocol import ErrorResponse
13 from vllm.entrypoints.serve.utils.api_utils import validate_json_request
14 from vllm.logger import init_logger
@@ -95,3 +96,25 @@ async def render_completion(request: CompletionRequest, raw_request: Request):
96 return JSONResponse(content=result.model_dump(), status_code=result.error.code)
97
98 return JSONResponse(content=[item.model_dump() for item in result])
99 +
100 +
101 +@router.post(
102 + "/v1/responses/render",
103 + dependencies=[Depends(validate_json_request)],
104 + response_model=GenerateRequest,
105 + responses={
106 + HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
107 + HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
108 + HTTPStatus.NOT_IMPLEMENTED.value: {"model": ErrorResponse},
109 + HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
110 + },
111 +)
112 +async def render_responses(request: ResponsesRequest, raw_request: Request):
113 + handler = render(raw_request)
114 + if handler is None:
115 + raise NotImplementedError("The model does not support Responses Render API")
116 +
117 + result = await handler.render_responses_request(request)
118 + if isinstance(result, ErrorResponse):
119 + return JSONResponse(content=result.model_dump(), status_code=result.error.code)
120 + return JSONResponse(content=result.model_dump())
vllm/entrypoints/scale_out/render/serving.py
+66
@@ -1,5 +1,7 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +from typing import TYPE_CHECKING
4 +
5 from vllm.entrypoints.anthropic.protocol import AnthropicMessagesRequest
6 from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
7 from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
@@ -8,6 +10,7 @@ from vllm.entrypoints.openai.models.serving import (
10 OpenAIModelRegistry,
11 OpenAIServingModels,
12 )
13 +from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
14 from vllm.entrypoints.scale_out.token_in_token_out.mm_features import (
15 extract_mm_features,
16 )
@@ -31,6 +34,9 @@ from vllm.utils import random_uuid
34
35 logger = init_logger(__name__)
36
37 +if TYPE_CHECKING:
38 + from vllm.entrypoints.mcp.tool_server import ToolServer
39 +
40
41 class ServingRender(BaseServing):
42 def __init__(
@@ -39,6 +45,7 @@ class ServingRender(BaseServing):
45 online_renderer: "OnlineRenderer",
46 *,
47 request_logger: RequestLogger | None = None,
48 + tool_server: "ToolServer | None" = None,
49 ) -> None:
50 super().__init__(
51 models=models,
@@ -47,6 +54,7 @@ class ServingRender(BaseServing):
54 )
55
56 self.online_renderer = online_renderer
57 + self.tool_server = tool_server
58
59 self._merge_inline_system = (
60 AnthropicServingMessages._detect_merge_inline_system(
@@ -229,6 +237,64 @@ class ServingRender(BaseServing):
237
238 return generate_requests
239
240 + async def render_responses_request(
241 + self,
242 + request: ResponsesRequest,
243 + ) -> GenerateRequest | ErrorResponse:
244 + error_check_ret = await self._check_model(request)
245 + if error_check_ret is not None:
246 + return error_check_ret
247 + if request.previous_response_id is not None:
248 + return self.create_error_response(
249 + message=(
250 + "previous_response_id is not supported by the stateless "
251 + "render endpoint."
252 + ),
253 + err_type="invalid_request_error",
254 + param="previous_response_id",
255 + )
256 +
257 + result = await self.online_renderer.render_responses(
258 + request,
259 + previous_messages=None,
260 + previous_response_outputs=None,
261 + tool_server=self.tool_server,
262 + skip_mm_cache=True,
263 + )
264 + if isinstance(result, ErrorResponse):
265 + return result
266 +
267 + engine_input = result.engine_input
268 + prompt_components = extract_prompt_components(self.model_config, engine_input)
269 + token_ids = prompt_components.token_ids
270 + if not token_ids:
271 + return self.create_error_response("No token_ids rendered")
272 +
273 + input_length = extract_prompt_len(self.model_config, engine_input)
274 + max_tokens = get_max_tokens(
275 + self.model_config.max_model_len,
276 + request.max_output_tokens,
277 + input_length,
278 + self.default_sampling_params,
279 + self.override_max_tokens,
280 + truncate_prompt_tokens=(-1 if request.truncation != "disabled" else None),
281 + )
282 + params = request.to_sampling_params(max_tokens, self.default_sampling_params)
283 +
284 + return GenerateRequest(
285 + request_id=request.request_id,
286 + token_ids=list(token_ids),
287 + features=self._extract_mm_features(engine_input),
288 + sampling_params=params,
289 + model=request.model,
290 + stream=bool(request.stream),
291 + cache_salt=request.cache_salt,
292 + priority=request.priority,
293 + kv_transfer_params=request.kv_transfer_params,
294 + ec_transfer_params=request.ec_transfer_params,
295 + token_offsets=engine_input.get("prompt_token_offsets"),
296 + )
297 +
298 def _placeholder_metadata_fields(self, modality: str) -> set[str]:
299 """Fields the EC consumer still requires after embeddings transfer."""
300 if self._placeholder_metadata_parser_failed:
vllm/renderers/online_renderer.py
+267 -2
@@ -1,13 +1,19 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +import time
4 from collections.abc import Sequence
5 +from dataclasses import dataclass
6 from http import HTTPStatus
5 -from typing import Any
7 +from typing import TYPE_CHECKING, Any, TypeAlias
8
9 +from openai.types.responses import ResponseFunctionToolCall, ResponseOutputItem
10 +from openai.types.responses.tool import Mcp, Tool
11 from openai_harmony import Message as OpenAIMessage
12 +from openai_harmony import ToolNamespaceConfig
13
14 from vllm.config import ModelConfig
15 from vllm.entrypoints.chat_utils import (
16 + ChatCompletionMessageParam,
17 ChatTemplateContentFormatOption,
18 ConversationMessage,
19 )
@@ -19,12 +25,24 @@ from vllm.entrypoints.openai.completion.protocol import (
25 CompletionRequest,
26 )
27 from vllm.entrypoints.openai.parser.harmony_utils import (
28 + BUILTIN_TOOL_TO_MCP_SERVER_LABEL,
29 build_harmony_preamble,
30 extract_instructions_from_messages,
31 + get_user_message,
32 + has_custom_tools,
33 parse_chat_inputs_to_harmony_messages,
34 render_for_completion,
35 )
36 +from vllm.entrypoints.openai.responses.harmony import (
37 + construct_harmony_previous_input_messages,
38 + response_input_to_harmony,
39 +)
40 from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
41 +from vllm.entrypoints.openai.responses.utils import (
42 + construct_input_messages,
43 + construct_tool_dicts,
44 + extract_tool_types,
45 +)
46 from vllm.entrypoints.serve import create_error_response
47 from vllm.entrypoints.serve.engine.protocol import ErrorResponse
48 from vllm.entrypoints.serve.utils.request_logger import RequestLogger
@@ -32,11 +50,12 @@ from vllm.inputs import (
50 EngineInput,
51 PromptType,
52 SingletonPrompt,
53 + TokensPrompt,
54 tokens_input,
55 )
56 from vllm.logger import init_logger
57 from vllm.parser import Parser, ParserManager
39 -from vllm.renderers import BaseRenderer, ChatParams, merge_kwargs
58 +from vllm.renderers import BaseRenderer, ChatParams, TokenizeParams, merge_kwargs
59 from vllm.renderers.inputs.preprocess import (
60 parse_model_prompt,
61 prompt_to_seq,
@@ -46,6 +65,39 @@ from vllm.utils.mistral import mt as _mt
65
66 logger = init_logger(__name__)
67
68 +if TYPE_CHECKING:
69 + from vllm.entrypoints.mcp.tool_server import ToolServer
70 +
71 +
72 +ResponsesPreviousMessages: TypeAlias = (
73 + list[ChatCompletionMessageParam] | list[OpenAIMessage]
74 +)
75 +
76 +
77 +@dataclass(frozen=True)
78 +class ResponsesRenderResult:
79 + messages: ResponsesPreviousMessages
80 + engine_input: EngineInput
81 +
82 +
83 +def _extract_allowed_tools_from_mcp_requests(
84 + tools: list[Tool],
85 +) -> dict[str, list[str] | None]:
86 + allowed_tools_map: dict[str, list[str] | None] = {}
87 + for tool in tools:
88 + if not isinstance(tool, Mcp):
89 + continue
90 +
91 + allowed_tools = None
92 + if isinstance(tool.allowed_tools, list):
93 + allowed_tools = tool.allowed_tools
94 + elif tool.allowed_tools is not None:
95 + allowed_tools = tool.allowed_tools.tool_names
96 + if allowed_tools is not None and "*" in allowed_tools:
97 + allowed_tools = None
98 + allowed_tools_map[tool.server_label] = allowed_tools
99 + return allowed_tools_map
100 +
101
102 def _reused_prompt_token_ids(request: Any) -> list[int] | None:
103 """Pop prompt token ids forwarded for decode-side reuse, if any.
@@ -217,6 +269,219 @@ class OnlineRenderer:
269
270 return conversation, engine_inputs
271
272 + async def render_responses(
273 + self,
274 + request: ResponsesRequest,
275 + *,
276 + previous_messages: ResponsesPreviousMessages | None = None,
277 + previous_response_outputs: list[ResponseOutputItem] | None = None,
278 + tool_server: "ToolServer | None" = None,
279 + skip_mm_cache: bool = False,
280 + ) -> ResponsesRenderResult | ErrorResponse:
281 + """Render a Responses request using only explicitly supplied history."""
282 + template_error = self.validate_chat_template(
283 + request_chat_template=None,
284 + chat_template_kwargs=request.chat_template_kwargs,
285 + trust_request_chat_template=self.trust_request_chat_template,
286 + )
287 + if template_error is not None:
288 + return template_error
289 +
290 + if self.use_harmony:
291 + return self._render_responses_with_harmony(
292 + request,
293 + previous_messages=previous_messages,
294 + previous_response_outputs=previous_response_outputs,
295 + tool_server=tool_server,
296 + )
297 +
298 + if previous_messages is not None and any(
299 + isinstance(message, OpenAIMessage) for message in previous_messages
300 + ):
301 + return self.create_error_response(
302 + "Non-Harmony Responses history must use chat messages.",
303 + err_type="invalid_request_error",
304 + param="previous_response_id",
305 + )
306 +
307 + tool_dicts = construct_tool_dicts(
308 + request.tools,
309 + request.tool_choice,
310 + exclude_tools_when_tool_choice_none=(
311 + self.exclude_tools_when_tool_choice_none
312 + ),
313 + )
314 + messages = construct_input_messages(
315 + request_instructions=request.instructions,
316 + request_input=request.input,
317 + prev_msg=list(previous_messages) if previous_messages is not None else None,
318 + prev_response_output=(
319 + list(previous_response_outputs)
320 + if previous_response_outputs is not None
321 + else None
322 + ),
323 + )
324 + chat_template_kwargs = (
325 + request.build_chat_params(
326 + self.chat_template,
327 + self.chat_template_content_format,
328 + )
329 + .with_defaults(self.default_chat_template_kwargs)
330 + .chat_template_kwargs
331 + )
332 + _, engine_inputs = await self.preprocess_chat(
333 + request,
334 + messages,
335 + default_template=self.chat_template,
336 + default_template_content_format=self.chat_template_content_format,
337 + default_template_kwargs=chat_template_kwargs,
338 + tool_dicts=tool_dicts,
339 + parser=self.parser,
340 + skip_mm_cache=skip_mm_cache,
341 + )
342 + return self._responses_render_result(messages, engine_inputs)
343 +
344 + def _render_responses_with_harmony(
345 + self,
346 + request: ResponsesRequest,
347 + *,
348 + previous_messages: ResponsesPreviousMessages | None,
349 + previous_response_outputs: list[ResponseOutputItem] | None,
350 + tool_server: "ToolServer | None",
351 + ) -> ResponsesRenderResult | ErrorResponse:
352 + if self.parser is not None:
353 + # HarmonyParser doesn't need chat_template_kwargs
354 + # TODO: Unify adjust_request() call with non-harmony branch
355 + self.parser(
356 + self.renderer.get_tokenizer(),
357 + request.tools,
358 + model_config=self.model_config,
359 + ).adjust_request(request=request)
360 +
361 + if previous_messages is not None and any(
362 + not isinstance(message, OpenAIMessage) for message in previous_messages
363 + ):
364 + return self.create_error_response(
365 + "Harmony Responses history must use Harmony messages.",
366 + err_type="invalid_request_error",
367 + param="previous_response_id",
368 + )
369 +
370 + messages: list[OpenAIMessage] = []
371 + request_input = request.input
372 + if previous_messages is None:
373 + tool_types = extract_tool_types(request.tools)
374 + with_custom_tools = has_custom_tools(tool_types)
375 + instructions = request.instructions
376 + if instructions is None and isinstance(request_input, list):
377 + instructions, request_input = extract_instructions_from_messages(
378 + request_input
379 + )
380 + descriptions = self._get_harmony_builtin_tool_descriptions(
381 + request.tools,
382 + tool_types,
383 + tool_server,
384 + )
385 + messages.extend(
386 + build_harmony_preamble(
387 + instructions=instructions,
388 + tools=request.tools if with_custom_tools else None,
389 + reasoning_effort=(
390 + request.reasoning.effort if request.reasoning else None
391 + ),
392 + with_custom_tools=with_custom_tools,
393 + **descriptions,
394 + )
395 + )
396 + messages.extend(construct_harmony_previous_input_messages(request))
397 + else:
398 + messages.extend(previous_messages)
399 +
400 + previous_outputs = list(previous_response_outputs or ())
401 + try:
402 + if isinstance(request_input, str):
403 + if request_input or not request.previous_input_messages:
404 + messages.append(get_user_message(request_input))
405 + else:
406 + for response_message in request_input:
407 + new_message = response_input_to_harmony(
408 + response_message,
409 + previous_outputs,
410 + )
411 + if new_message is not None:
412 + messages.append(new_message)
413 + if isinstance(response_message, ResponseFunctionToolCall):
414 + previous_outputs.append(response_message)
415 + except ValueError as exc:
416 + return self.create_error_response(
417 + str(exc),
418 + err_type="invalid_request_error",
419 + param="input",
420 + )
421 +
422 + return ResponsesRenderResult(
423 + messages=messages,
424 + engine_input=self.render_responses_harmony_messages(
425 + messages,
426 + cache_salt=request.cache_salt,
427 + tok_params=request.build_tok_params(self.model_config),
428 + ),
429 + )
430 +
431 + def _get_harmony_builtin_tool_descriptions(
432 + self,
433 + tools: list[Tool],
434 + tool_types: set[str],
435 + tool_server: "ToolServer | None",
436 + ) -> dict[str, ToolNamespaceConfig | None]:
437 + allowed_tools = _extract_allowed_tools_from_mcp_requests(tools)
438 + descriptions: dict[str, ToolNamespaceConfig | None] = {}
439 + for server_name, request_name in BUILTIN_TOOL_TO_MCP_SERVER_LABEL.items():
440 + description = (
441 + tool_server.get_tool_description(
442 + server_name,
443 + allowed_tools.get(request_name),
444 + )
445 + if request_name in tool_types
446 + and tool_server is not None
447 + and tool_server.has_tool(server_name)
448 + else None
449 + )
450 + descriptions[f"{server_name}_description"] = description
451 + return descriptions
452 +
453 + def render_responses_harmony_messages(
454 + self,
455 + messages: list[OpenAIMessage],
456 + *,
457 + cache_salt: str | None,
458 + tok_params: TokenizeParams | None = None,
459 + ) -> EngineInput:
460 + arrival_time = time.time()
461 + prompt = TokensPrompt(prompt_token_ids=render_for_completion(messages))
462 + if tok_params is not None:
463 + tok_params.apply_post_tokenization(
464 + self.renderer.tokenizer,
465 + prompt,
466 + )
467 + engine_input = tokens_input(prompt["prompt_token_ids"], cache_salt=cache_salt)
468 + engine_input["arrival_time"] = arrival_time
469 + return engine_input
470 +
471 + def _responses_render_result(
472 + self,
473 + messages: list[ChatCompletionMessageParam],
474 + engine_inputs: list[EngineInput],
475 + ) -> ResponsesRenderResult | ErrorResponse:
476 + if len(engine_inputs) != 1:
477 + return self.create_error_response(
478 + f"Expected exactly 1 engine prompt, got {len(engine_inputs)}"
479 + )
480 + return ResponsesRenderResult(
481 + messages=messages,
482 + engine_input=engine_inputs[0],
483 + )
484 +
485 def _make_request_with_harmony(
486 self,
487 request: ChatCompletionRequest,