[Render] Add reasoning/tool parsing to /derender + fix byte-fallback FFFD (#45919)
Signed-off-by: aoshen524 <aoshen524@gmail.com> Co-authored-by: Martin Hickey <martin.hickey@ie.ibm.com>
aoshen02 committed
Jun 21, 2026 at 07:43 UTC
f57ac274b24cbb5a4e079a529175eef5c0745606
4 files changed
+601
-37
tests/entrypoints/serve/render/test_derender.py
+436
@@ -8,6 +8,7 @@ import pytest
8
import pytest_asyncio
9
10
from tests.utils import RemoteLaunchRenderServer
11
+from vllm.tokenizers import get_tokenizer
12
13
MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
14
@@ -486,3 +487,438 @@ async def test_derender_completion_kv_transfer_params_passthrough(client):
487
)
488
assert response.status_code == 200
489
assert response.json()["kv_transfer_params"] == kv
490
+
491
+
492
+# ---------------------------------------------------------------------------
493
+# E2E: render -> derender roundtrip with parser (reasoning + tool calls)
494
+# ---------------------------------------------------------------------------
495
+
496
+PARSER_MODEL = "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B"
497
+
498
+_E2E_TOOLS = [
499
+ {
500
+ "type": "function",
501
+ "function": {
502
+ "name": "get_weather",
503
+ "description": "Get weather for a city",
504
+ "parameters": {
505
+ "type": "object",
506
+ "properties": {"city": {"type": "string"}},
507
+ },
508
+ },
509
+ }
510
+]
511
+
512
+
513
+@pytest.fixture(scope="module")
514
+def parser_server():
515
+ args = [
516
+ "--enable-auto-tool-choice",
517
+ "--tool-call-parser",
518
+ "hermes",
519
+ "--reasoning-parser",
520
+ "deepseek_r1",
521
+ ]
522
+ with RemoteLaunchRenderServer(PARSER_MODEL, args) as remote_server:
523
+ yield remote_server
524
+
525
+
526
+@pytest_asyncio.fixture
527
+async def parser_client(parser_server):
528
+ async with httpx.AsyncClient(
529
+ base_url=parser_server.url_for(""), timeout=60.0
530
+ ) as http_client:
531
+ yield http_client
532
+
533
+
534
+@pytest.fixture(scope="module")
535
+def parser_tokenizer():
536
+ return get_tokenizer(PARSER_MODEL)
537
+
538
+
539
+def _encode(tokenizer, text: str) -> list[int]:
540
+ return tokenizer.encode(text, add_special_tokens=False)
541
+
542
+
543
+def _decoded(tokenizer, token_ids: list[int]) -> str:
544
+ return tokenizer.decode(token_ids, skip_special_tokens=True)
545
+
546
+
547
+def _require_markers_survive(tokenizer, text: str, *markers: str) -> list[int]:
548
+ """Encode text and skip the test if any marker is lost in roundtrip."""
549
+ ids = _encode(tokenizer, text)
550
+ decoded = tokenizer.decode(ids, skip_special_tokens=False)
551
+ for m in markers:
552
+ if m not in decoded:
553
+ pytest.skip(f"Marker {m!r} lost in encode->decode roundtrip")
554
+ return ids
555
+
556
+
557
+async def _e2e_render_chat(
558
+ client: httpx.AsyncClient,
559
+ model: str,
560
+ messages: list[dict],
561
+) -> dict:
562
+ resp = await client.post(
563
+ "/v1/chat/completions/render",
564
+ json={"model": model, "messages": messages},
565
+ )
566
+ assert resp.status_code == 200, resp.text
567
+ return resp.json()
568
+
569
+
570
+def _e2e_generate_response(
571
+ token_ids: list[int],
572
+ request_id: str = "chatcmpl-e2e-test",
573
+) -> dict:
574
+ return {
575
+ "request_id": request_id,
576
+ "choices": [
577
+ {
578
+ "index": 0,
579
+ "token_ids": token_ids,
580
+ "finish_reason": "stop",
581
+ }
582
+ ],
583
+ }
584
+
585
+
586
+@pytest.mark.asyncio
587
+async def test_e2e_plain_roundtrip(parser_client, parser_tokenizer):
588
+ """Plain text without reasoning markers roundtrips correctly."""
589
+ messages = [{"role": "user", "content": "What is 2+2?"}]
590
+ gen_req = await _e2e_render_chat(parser_client, PARSER_MODEL, messages)
591
+
592
+ answer = "The answer is four."
593
+ output_ids = _encode(parser_tokenizer, answer)
594
+ expected = _decoded(parser_tokenizer, output_ids)
595
+
596
+ resp = await parser_client.post(
597
+ "/v1/chat/completions/derender",
598
+ json={
599
+ "model": PARSER_MODEL,
600
+ "generate_response": _e2e_generate_response(output_ids),
601
+ "prompt_tokens": len(gen_req["token_ids"]),
602
+ },
603
+ )
604
+ assert resp.status_code == 200, resp.text
605
+ content = resp.json()["choices"][0]["message"]["content"]
606
+ assert content == expected
607
+
608
+
609
+@pytest.mark.asyncio
610
+async def test_e2e_token_identity(parser_client, parser_tokenizer):
611
+ """encode(derender(token_ids)) == token_ids (RL invariant)."""
612
+ messages = [{"role": "user", "content": "Hi"}]
613
+ gen_req = await _e2e_render_chat(parser_client, PARSER_MODEL, messages)
614
+
615
+ answer = "Hello! How can I help?"
616
+ output_ids = _encode(parser_tokenizer, answer)
617
+
618
+ resp = await parser_client.post(
619
+ "/v1/chat/completions/derender",
620
+ json={
621
+ "model": PARSER_MODEL,
622
+ "generate_response": _e2e_generate_response(output_ids),
623
+ "prompt_tokens": len(gen_req["token_ids"]),
624
+ },
625
+ )
626
+ assert resp.status_code == 200
627
+ content = resp.json()["choices"][0]["message"]["content"]
628
+ re_encoded = _encode(parser_tokenizer, content)
629
+ assert output_ids == re_encoded
630
+
631
+
632
+@pytest.mark.asyncio
633
+async def test_e2e_non_ascii_roundtrip(parser_client, parser_tokenizer):
634
+ """CJK + emoji roundtrip without U+FFFD."""
635
+ messages = [{"role": "user", "content": "Reply in Chinese"}]
636
+ gen_req = await _e2e_render_chat(parser_client, PARSER_MODEL, messages)
637
+
638
+ answer = "你好世界 😀"
639
+ output_ids = _encode(parser_tokenizer, answer)
640
+
641
+ resp = await parser_client.post(
642
+ "/v1/chat/completions/derender",
643
+ json={
644
+ "model": PARSER_MODEL,
645
+ "generate_response": _e2e_generate_response(output_ids),
646
+ "prompt_tokens": len(gen_req["token_ids"]),
647
+ },
648
+ )
649
+ assert resp.status_code == 200
650
+ content = resp.json()["choices"][0]["message"]["content"]
651
+ assert "�" not in content
652
+
653
+
654
+@pytest.mark.asyncio
655
+async def test_e2e_parsed_reasoning(parser_client, parser_tokenizer):
656
+ """<think>...</think> splits into reasoning + content."""
657
+ messages = [{"role": "user", "content": "What is 2+3?"}]
658
+ gen_req = await _e2e_render_chat(parser_client, PARSER_MODEL, messages)
659
+
660
+ reasoning_text = "The user wants 2 plus 3. That is 5."
661
+ answer_text = "The answer is 5."
662
+ output_text = f"<think>{reasoning_text}</think>{answer_text}"
663
+ output_ids = _require_markers_survive(parser_tokenizer, output_text, "</think>")
664
+
665
+ resp = await parser_client.post(
666
+ "/v1/chat/completions/derender",
667
+ json={
668
+ "model": PARSER_MODEL,
669
+ "generate_response": _e2e_generate_response(output_ids),
670
+ "prompt_tokens": len(gen_req["token_ids"]),
671
+ "chat_request": {
672
+ "model": PARSER_MODEL,
673
+ "messages": messages,
674
+ "include_reasoning": True,
675
+ },
676
+ },
677
+ )
678
+ assert resp.status_code == 200, resp.text
679
+ msg = resp.json()["choices"][0]["message"]
680
+ assert msg["reasoning"] is not None
681
+ assert reasoning_text in msg["reasoning"]
682
+ assert answer_text in msg["content"]
683
+ assert "<think>" not in msg["content"]
684
+
685
+
686
+@pytest.mark.asyncio
687
+async def test_e2e_parsed_tool_call(parser_client, parser_tokenizer):
688
+ """<tool_call> extracted into tool_calls field."""
689
+ messages = [{"role": "user", "content": "Weather in Paris?"}]
690
+ gen_req = await _e2e_render_chat(parser_client, PARSER_MODEL, messages)
691
+
692
+ output_text = (
693
+ "<think>Let me check the weather.</think>"
694
+ '<tool_call>\n{"name": "get_weather", '
695
+ '"arguments": {"city": "Paris"}}\n</tool_call>'
696
+ )
697
+ output_ids = _require_markers_survive(
698
+ parser_tokenizer,
699
+ output_text,
700
+ "</think>",
701
+ "<tool_call>",
702
+ "</tool_call>",
703
+ )
704
+
705
+ resp = await parser_client.post(
706
+ "/v1/chat/completions/derender",
707
+ json={
708
+ "model": PARSER_MODEL,
709
+ "generate_response": _e2e_generate_response(output_ids),
710
+ "prompt_tokens": len(gen_req["token_ids"]),
711
+ "chat_request": {
712
+ "model": PARSER_MODEL,
713
+ "messages": messages,
714
+ "tools": _E2E_TOOLS,
715
+ "tool_choice": "auto",
716
+ },
717
+ },
718
+ )
719
+ assert resp.status_code == 200, resp.text
720
+ choice = resp.json()["choices"][0]
721
+ assert choice["message"]["tool_calls"]
722
+ assert choice["message"]["tool_calls"][0]["function"]["name"] == "get_weather"
723
+
724
+
725
+@pytest.mark.asyncio
726
+async def test_e2e_parsed_reasoning_and_tool_call(parser_client, parser_tokenizer):
727
+ """Reasoning + tool call in the same output."""
728
+ messages = [{"role": "user", "content": "Weather in Paris?"}]
729
+ gen_req = await _e2e_render_chat(parser_client, PARSER_MODEL, messages)
730
+
731
+ reasoning_text = "I should look up the weather."
732
+ tool_text = (
733
+ '<tool_call>\n{"name": "get_weather", '
734
+ '"arguments": {"city": "Paris"}}\n</tool_call>'
735
+ )
736
+ output_text = f"<think>{reasoning_text}</think>{tool_text}"
737
+ output_ids = _require_markers_survive(
738
+ parser_tokenizer, output_text, "</think>", "<tool_call>"
739
+ )
740
+
741
+ resp = await parser_client.post(
742
+ "/v1/chat/completions/derender",
743
+ json={
744
+ "model": PARSER_MODEL,
745
+ "generate_response": _e2e_generate_response(output_ids),
746
+ "prompt_tokens": len(gen_req["token_ids"]),
747
+ "chat_request": {
748
+ "model": PARSER_MODEL,
749
+ "messages": messages,
750
+ "tools": _E2E_TOOLS,
751
+ "tool_choice": "auto",
752
+ "include_reasoning": True,
753
+ },
754
+ },
755
+ )
756
+ assert resp.status_code == 200, resp.text
757
+ choice = resp.json()["choices"][0]
758
+ assert choice["message"]["reasoning"] is not None
759
+ assert reasoning_text in choice["message"]["reasoning"]
760
+ assert choice["message"]["tool_calls"]
761
+
762
+
763
+@pytest.mark.asyncio
764
+async def test_e2e_no_chat_request_fallback(parser_client, parser_tokenizer):
765
+ """Without chat_request, derender falls back to plain detokenization."""
766
+ messages = [{"role": "user", "content": "Hello"}]
767
+ gen_req = await _e2e_render_chat(parser_client, PARSER_MODEL, messages)
768
+
769
+ answer = "Hi there!"
770
+ output_ids = _encode(parser_tokenizer, answer)
771
+
772
+ resp = await parser_client.post(
773
+ "/v1/chat/completions/derender",
774
+ json={
775
+ "model": PARSER_MODEL,
776
+ "generate_response": _e2e_generate_response(output_ids),
777
+ "prompt_tokens": len(gen_req["token_ids"]),
778
+ },
779
+ )
780
+ assert resp.status_code == 200
781
+ content = resp.json()["choices"][0]["message"]["content"]
782
+ assert "Hi" in content
783
+
784
+
785
+# ---------------------------------------------------------------------------
786
+# E2E: HarmonyParser + GPT-OSS
787
+# ---------------------------------------------------------------------------
788
+
789
+HARMONY_MODEL = "openai/gpt-oss-20b"
790
+
791
+
792
+def _ensure_harmony_vocab():
793
+ """Pre-cache the o200k_base BPE file needed by openai-harmony.
794
+
795
+ The Rust tiktoken-rs backend downloads from Azure Blob Storage, which
796
+ may be unreachable in some environments. When the cache is cold we
797
+ fetch the file ourselves and place it in ``/tmp/tiktoken-rs-cache/``
798
+ using the SHA-1(URL) filename that tiktoken-rs expects.
799
+ """
800
+ import hashlib
801
+ import urllib.request
802
+ from pathlib import Path
803
+
804
+ url = "https://openaipublic.blob.core.windows.net/encodings/o200k_base.tiktoken"
805
+ cache_dir = Path("/tmp/tiktoken-rs-cache")
806
+ cache_key = hashlib.sha1(url.encode()).hexdigest()
807
+ cache_file = cache_dir / cache_key
808
+ if not cache_file.exists():
809
+ cache_dir.mkdir(parents=True, exist_ok=True)
810
+ urllib.request.urlretrieve(url, cache_file)
811
+
812
+
813
+@pytest.fixture(scope="module")
814
+def harmony_server():
815
+ _ensure_harmony_vocab()
816
+ args = [
817
+ "--trust-remote-code",
818
+ "--enable-auto-tool-choice",
819
+ "--tool-call-parser",
820
+ "openai",
821
+ "--reasoning-parser",
822
+ "openai_gptoss",
823
+ ]
824
+ with RemoteLaunchRenderServer(HARMONY_MODEL, args) as remote_server:
825
+ yield remote_server
826
+
827
+
828
+@pytest_asyncio.fixture
829
+async def harmony_client(harmony_server):
830
+ async with httpx.AsyncClient(
831
+ base_url=harmony_server.url_for(""), timeout=60.0
832
+ ) as http_client:
833
+ yield http_client
834
+
835
+
836
+@pytest.fixture(scope="module")
837
+def harmony_tokenizer():
838
+ return get_tokenizer(HARMONY_MODEL, trust_remote_code=True)
839
+
840
+
841
+def _harmony_extract_assistant_ids(
842
+ tokenizer, assistant_msg: dict, user_content: str = "test"
843
+) -> list[int]:
844
+ """Extract assistant token IDs via apply_chat_template diff."""
845
+ prompt = [{"role": "user", "content": user_content}]
846
+ full = prompt + [assistant_msg]
847
+ text_prompt = tokenizer.apply_chat_template(
848
+ prompt, add_generation_prompt=True, tokenize=False
849
+ )
850
+ text_full = tokenizer.apply_chat_template(
851
+ full, add_generation_prompt=False, tokenize=False
852
+ )
853
+ prompt_ids = tokenizer.encode(text_prompt)
854
+ full_ids = tokenizer.encode(text_full)
855
+ assistant_ids = list(full_ids[len(prompt_ids) :])
856
+ if not assistant_ids:
857
+ pytest.skip("Could not extract assistant tokens for Harmony")
858
+ return assistant_ids
859
+
860
+
861
+@pytest.mark.asyncio
862
+async def test_e2e_harmony_plain_roundtrip(harmony_client, harmony_tokenizer):
863
+ """GPT-OSS content-only roundtrip."""
864
+ messages = [{"role": "user", "content": "What is 2+2?"}]
865
+ gen_req = await _e2e_render_chat(harmony_client, HARMONY_MODEL, messages)
866
+
867
+ assistant_msg = {"role": "assistant", "content": "Four."}
868
+ output_ids = _harmony_extract_assistant_ids(harmony_tokenizer, assistant_msg)
869
+
870
+ resp = await harmony_client.post(
871
+ "/v1/chat/completions/derender",
872
+ json={
873
+ "model": HARMONY_MODEL,
874
+ "generate_response": _e2e_generate_response(output_ids),
875
+ "prompt_tokens": len(gen_req["token_ids"]),
876
+ "chat_request": {
877
+ "model": HARMONY_MODEL,
878
+ "messages": messages,
879
+ },
880
+ },
881
+ )
882
+ assert resp.status_code == 200, resp.text
883
+ content = resp.json()["choices"][0]["message"]["content"]
884
+ assert content is not None and len(content) > 0
885
+ assert "Four" in content
886
+
887
+
888
+@pytest.mark.asyncio
889
+async def test_e2e_harmony_reasoning(harmony_client, harmony_tokenizer):
890
+ """GPT-OSS reasoning: analysis channel extracted."""
891
+ messages = [{"role": "user", "content": "Add 2 and 3."}]
892
+ gen_req = await _e2e_render_chat(harmony_client, HARMONY_MODEL, messages)
893
+
894
+ reasoning_text = "The user wants 2 plus 3."
895
+ answer_text = "The answer is 5."
896
+ assistant_msg = {
897
+ "role": "assistant",
898
+ "thinking": reasoning_text,
899
+ "content": answer_text,
900
+ }
901
+ output_ids = _harmony_extract_assistant_ids(harmony_tokenizer, assistant_msg)
902
+
903
+ decoded = harmony_tokenizer.decode(output_ids)
904
+ if reasoning_text not in decoded:
905
+ pytest.skip("Harmony template did not render thinking")
906
+
907
+ resp = await harmony_client.post(
908
+ "/v1/chat/completions/derender",
909
+ json={
910
+ "model": HARMONY_MODEL,
911
+ "generate_response": _e2e_generate_response(output_ids),
912
+ "prompt_tokens": len(gen_req["token_ids"]),
913
+ "chat_request": {
914
+ "model": HARMONY_MODEL,
915
+ "messages": messages,
916
+ "include_reasoning": True,
917
+ },
918
+ },
919
+ )
920
+ assert resp.status_code == 200, resp.text
921
+ msg = resp.json()["choices"][0]["message"]
922
+ assert msg["reasoning"] is not None
923
+ assert reasoning_text in msg["reasoning"]
924
+ assert answer_text in (msg["content"] or "")
vllm/entrypoints/openai/api_server.py
+1
-1
@@ -455,7 +455,7 @@ async def init_render_app_state(
455
enable_auto_tools=args.enable_auto_tool_choice,
456
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
457
tool_parser=args.tool_call_parser,
458
- reasoning_parser=args.structured_outputs_config.reasoning_parser,
458
+ reasoning_parser=args.reasoning_parser,
459
default_chat_template_kwargs=args.default_chat_template_kwargs,
460
log_error_stack=args.log_error_stack,
461
)
vllm/entrypoints/serve/disagg/protocol.py
+8
-4
@@ -219,10 +219,14 @@ class GenerateResponse(BaseModel):
219
220
221
class DerenderChatRequest(BaseModel):
222
- """Request for the /v1/chat/completions/derender endpoint.
222
+ """Request for the /v1/chat/completions/derender endpoint (non-streaming).
223
224
- Wraps a GenerateResponse and caller-supplied metadata needed to produce
225
- a fully-formed ChatCompletionResponse without a GPU.
224
+ Wraps a complete GenerateResponse and caller-supplied metadata needed to
225
+ produce a fully-formed ChatCompletionResponse without a GPU.
226
+
227
+ Streaming derender would require a separate endpoint design with
228
+ incremental token delivery, ``OutputProcessor``-based detokenization,
229
+ and ``parser.parse_delta()`` instead of ``parser.parse()``.
230
"""
231
232
model: str
@@ -244,7 +248,7 @@ class DerenderChatRequest(BaseModel):
248
249
250
class DerenderCompletionRequest(BaseModel):
247
- """Request for the /v1/completions/derender endpoint.
251
+ """Request for the /v1/completions/derender endpoint (non-streaming).
252
253
Parallel to DerenderChatRequest but handles the multi-prompt completions
254
case: one GenerateResponse per prompt, mirroring the list[GenerateRequest]
vllm/entrypoints/serve/render/serving.py
+156
-32
@@ -27,6 +27,7 @@ from vllm.entrypoints.openai.completion.protocol import (
27
)
28
from vllm.entrypoints.openai.engine.protocol import (
29
ErrorResponse,
30
+ ToolCall,
31
UsageInfo,
32
)
33
from vllm.entrypoints.openai.engine.serving import resolve_token_id_placeholder
@@ -43,7 +44,6 @@ from vllm.entrypoints.serve.disagg.protocol import (
44
DerenderChatRequest,
45
DerenderCompletionRequest,
46
GenerateRequest,
46
- GenerateResponseChoice,
47
MultiModalFeatures,
48
PlaceholderRangeInfo,
49
)
@@ -76,21 +76,83 @@ from vllm.utils.mistral import mt as _mt
76
logger = init_logger(__name__)
77
78
79
+def _parse_token_id_placeholder(token: str) -> int | None:
80
+ """Extract token ID from a 'token_id:N' placeholder string."""
81
+ if not token.startswith("token_id:"):
82
+ return None
83
+ try:
84
+ return int(token[len("token_id:") :])
85
+ except ValueError:
86
+ return None
87
+
88
+
89
+def _correct_decoded_token(
90
+ token_id: int, context_token_ids: list[int], tokenizer: TokenizerLike
91
+) -> str:
92
+ """Use preceding tokens as context to fix U+FFFD from byte-fallback.
93
+
94
+ Mirrors LogprobsProcessor._correct_decoded_token in v1/engine/logprobs.py.
95
+ """
96
+ max_ctx = min(len(context_token_ids), 4)
97
+
98
+ for num_ctx in range(1, max_ctx + 1):
99
+ context = context_token_ids[-num_ctx:]
100
+ full_decoded = tokenizer.decode(context + [token_id])
101
+
102
+ if full_decoded.endswith("�"):
103
+ continue
104
+
105
+ clean_end = len(context)
106
+ for j in range(len(context) - 1, -1, -1):
107
+ if tokenizer.decode([context[j]]).endswith("�"):
108
+ clean_end = j
109
+ else:
110
+ break
111
+
112
+ clean_prefix = tokenizer.decode(context[:clean_end]) if clean_end > 0 else ""
113
+
114
+ if full_decoded.startswith(clean_prefix):
115
+ return full_decoded[len(clean_prefix) :]
116
+
117
+ common_len = 0
118
+ for a, b in zip(clean_prefix, full_decoded):
119
+ if a != b:
120
+ break
121
+ common_len += 1
122
+ return full_decoded[common_len:]
123
+
124
+ return ""
125
+
126
+
127
def _resolve_logprobs(
128
logprobs: ChatCompletionLogProbs, tokenizer: TokenizerLike
129
) -> ChatCompletionLogProbs:
82
- """Resolve all token_id:N placeholders in a ChatCompletionLogProbs object."""
130
+ """Resolve token_id:N placeholders in a ChatCompletionLogProbs object."""
131
if logprobs.content is None:
132
return logprobs
133
+
134
+ context_token_ids: list[int] = []
135
resolved_content = []
136
+
137
for entry in logprobs.content:
138
token_str, token_bytes = resolve_token_id_placeholder(entry.token, tokenizer)
139
+ sampled_id = _parse_token_id_placeholder(entry.token)
140
+
141
+ if token_str.endswith("�") and sampled_id is not None:
142
+ token_str = _correct_decoded_token(sampled_id, context_token_ids, tokenizer)
143
+ token_bytes = list(token_str.encode("utf-8"))
144
+
145
resolved_top = []
146
for top in entry.top_logprobs:
147
top_str, top_bytes = resolve_token_id_placeholder(top.token, tokenizer)
148
+ top_id = _parse_token_id_placeholder(top.token)
149
+ if top_str.endswith("�") and top_id is not None:
150
+ top_str = _correct_decoded_token(top_id, context_token_ids, tokenizer)
151
+ top_bytes = list(top_str.encode("utf-8"))
152
resolved_top.append(
153
top.model_copy(update={"token": top_str, "bytes": top_bytes})
154
)
155
+
156
resolved_content.append(
157
entry.model_copy(
158
update={
@@ -100,6 +162,10 @@ def _resolve_logprobs(
162
}
163
)
164
)
165
+
166
+ if sampled_id is not None:
167
+ context_token_ids.append(sampled_id)
168
+
169
return ChatCompletionLogProbs(content=resolved_content)
170
171
@@ -136,30 +202,6 @@ def _convert_chat_logprobs_to_completion_logprobs(
202
)
203
204
139
-def _build_chat_choice(
140
- choice: GenerateResponseChoice, tokenizer: TokenizerLike
141
-) -> ChatCompletionResponseChoice:
142
- """Detokenize and resolve logprobs for a single GenerateResponseChoice.
143
-
144
- Raises:
145
- ValueError: if choice.token_ids is empty or None.
146
- """
147
- if not choice.token_ids:
148
- raise ValueError(f"choice {choice.index} has empty or null token_ids")
149
- decoded_text = tokenizer.decode(choice.token_ids, skip_special_tokens=True)
150
- resolved_logprobs = (
151
- _resolve_logprobs(choice.logprobs, tokenizer)
152
- if choice.logprobs is not None
153
- else None
154
- )
155
- return ChatCompletionResponseChoice(
156
- index=choice.index,
157
- message=ChatMessage(role="assistant", content=decoded_text),
158
- logprobs=resolved_logprobs,
159
- finish_reason=choice.finish_reason,
160
- )
161
-
162
-
205
class OpenAIServingRender:
206
def __init__(
207
self,
@@ -536,9 +578,12 @@ class OpenAIServingRender:
578
) -> ChatCompletionResponse | ErrorResponse:
579
"""Postprocess a GenerateResponse into a ChatCompletionResponse.
580
539
- This is the symmetric inverse of render_chat_request: it detokenizes
540
- output token IDs, resolves token_id:N logprob placeholders, and
541
- formats the result as an OpenAI-compatible chat completion response.
581
+ Non-streaming only: expects the complete GenerateResponse with all
582
+ token IDs present. Uses ``parser.parse()`` for one-shot extraction.
583
+
584
+ When ``request.chat_request`` is provided, the parser splits the
585
+ output into (reasoning, content, tool_calls). Otherwise falls
586
+ back to plain detokenization.
587
"""
588
error_check_ret = await self._check_model(request)
589
if error_check_ret is not None:
@@ -546,11 +591,89 @@ class OpenAIServingRender:
591
592
tokenizer = self.renderer.get_tokenizer()
593
gen = request.generate_response
594
+ chat_request = request.chat_request
595
choices: list[ChatCompletionResponseChoice] = []
596
597
try:
598
for choice in gen.choices:
553
- choices.append(_build_chat_choice(choice, tokenizer))
599
+ if not choice.token_ids:
600
+ raise ValueError(
601
+ f"choice {choice.index} has empty or null token_ids"
602
+ )
603
+
604
+ resolved_logprobs = (
605
+ _resolve_logprobs(choice.logprobs, tokenizer)
606
+ if choice.logprobs is not None
607
+ else None
608
+ )
609
+
610
+ if self.parser is not None and chat_request is not None:
611
+ # Parser path: decode with special tokens preserved
612
+ # so the parser can see markers like </think>,
613
+ # <tool_call>, or Harmony channel tokens.
614
+ decoded_text = tokenizer.decode(
615
+ choice.token_ids, skip_special_tokens=False
616
+ )
617
+
618
+ chat_template_kwargs: dict[str, Any] = {}
619
+ if not self.use_harmony:
620
+ chat_template_kwargs = (
621
+ chat_request.build_chat_params(
622
+ self.chat_template,
623
+ self.chat_template_content_format,
624
+ )
625
+ .with_defaults(self.default_chat_template_kwargs)
626
+ .chat_template_kwargs
627
+ )
628
+
629
+ parser = self.parser(
630
+ tokenizer,
631
+ chat_request.tools,
632
+ chat_template_kwargs=chat_template_kwargs,
633
+ )
634
+ reasoning, content, tool_calls = parser.parse(
635
+ decoded_text,
636
+ chat_request,
637
+ enable_auto_tools=self.enable_auto_tools,
638
+ model_output_token_ids=choice.token_ids,
639
+ )
640
+
641
+ if not getattr(chat_request, "include_reasoning", True):
642
+ reasoning = None
643
+
644
+ tc_items = (
645
+ [
646
+ ToolCall(
647
+ id=random_uuid(),
648
+ function=tc,
649
+ )
650
+ for tc in tool_calls
651
+ ]
652
+ if tool_calls
653
+ else []
654
+ )
655
+
656
+ message = ChatMessage(
657
+ role="assistant",
658
+ reasoning=reasoning,
659
+ content=content,
660
+ tool_calls=tc_items,
661
+ )
662
+ else:
663
+ # No parser: plain detokenization.
664
+ decoded_text = tokenizer.decode(
665
+ choice.token_ids, skip_special_tokens=True
666
+ )
667
+ message = ChatMessage(role="assistant", content=decoded_text)
668
+
669
+ choices.append(
670
+ ChatCompletionResponseChoice(
671
+ index=choice.index,
672
+ message=message,
673
+ logprobs=resolved_logprobs,
674
+ finish_reason=choice.finish_reason,
675
+ )
676
+ )
677
except ValueError as exc:
678
return self.create_error_response(str(exc))
679
@@ -587,8 +710,9 @@ class OpenAIServingRender:
710
) -> CompletionResponse | ErrorResponse:
711
"""Postprocess a list of GenerateResponses into a CompletionResponse.
712
590
- Mirrors the multi-prompt completions case: one GenerateResponse per
591
- prompt, parallel to the list[GenerateRequest] from /v1/completions/render.
713
+ Non-streaming only. Mirrors the multi-prompt completions case: one
714
+ GenerateResponse per prompt, parallel to the list[GenerateRequest]
715
+ from /v1/completions/render.
716
"""
717
error_check_ret = await self._check_model(request)
718
if error_check_ret is not None: