test_serving_chat.py · tduncan/Nvidia-Optimization-Stack-VLLM

@tduncan / Nvidia-Optimization-Stack-VLLM

Code Commits Issues Pulls Cloud Agent CI/CD

main

py 3,043 lines 103 KB

Raw

1	# SPDX-License-Identifier: Apache-2.0
2	# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3	import asyncio
4	import json
5	from collections.abc import AsyncIterator
6	from contextlib import suppress
7	from dataclasses import dataclass, field
8	from typing import Any
9	from unittest.mock import AsyncMock, MagicMock
10
11	import pytest
12	import pytest_asyncio
13	from openai import OpenAI
14
15	from tests.entrypoints.openai.utils import (
16	accumulate_streaming_response,
17	verify_chat_response,
18	verify_harmony_messages,
19	)
20	from tests.utils import RemoteOpenAIServer
21	from vllm._aiter_ops import is_aiter_found_and_supported
22	from vllm.config import MultiModalConfig
23	from vllm.entrypoints.generate.base.protocol import (
24	DeltaMessage,
25	RequestResponseMetadata,
26	)
27	from vllm.entrypoints.generate.base.serving import build_per_request_timing_metrics
28	from vllm.entrypoints.openai.chat_completion.protocol import (
29	ChatCompletionRequest,
30	ChatCompletionResponse,
31	)
32	from vllm.entrypoints.openai.chat_completion.serving import (
33	OpenAIServingChat,
34	_get_mm_token_counts,
35	_make_completion_tokens_details,
36	_make_prompt_tokens_details,
37	)
38	from vllm.entrypoints.openai.models.serving import (
39	BaseModelPath,
40	OpenAIModelRegistry,
41	OpenAIServingModels,
42	)
43	from vllm.entrypoints.openai.parser.harmony_utils import get_encoding
44	from vllm.entrypoints.serve.engine.protocol import ErrorResponse
45	from vllm.exceptions import QueueOverflowError, VLLMValidationError
46	from vllm.inputs import TokensPrompt
47	from vllm.logprobs import Logprob
48	from vllm.multimodal.inputs import PlaceholderRange
49	from vllm.outputs import CompletionOutput, RequestOutput
50	from vllm.parser import HarmonyParser
51	from vllm.renderers.hf import HfRenderer
52	from vllm.renderers.mistral import MistralRenderer
53	from vllm.renderers.online_renderer import OnlineRenderer
54	from vllm.tokenizers import get_tokenizer
55	from vllm.tokenizers.mistral import MistralTokenizer
56	from vllm.tokenizers.registry import cached_tokenizer_from_config
57	from vllm.v1.engine.async_llm import AsyncLLM
58	from vllm.v1.metrics.stats import RequestStateStats
59
60	GPT_OSS_MODEL_NAME = "openai/gpt-oss-20b"
61	GPT_OSS_SPECULATOR_NAME = "RedHatAI/gpt-oss-20b-speculator.eagle3"
62	_PER_REQUEST_STATS = RequestStateStats(
63	queued_ts=1.0,
64	scheduled_ts=1.5,
65	first_token_ts=2.0,
66	last_token_ts=3.0,
67	num_generation_tokens=2,
68	)
69
70
71	@pytest.fixture(scope="module")
72	def monkeypatch_module():
73	from _pytest.monkeypatch import MonkeyPatch
74
75	mpatch = MonkeyPatch()
76	yield mpatch
77	mpatch.undo()
78
79
80	@pytest.fixture(
81	scope="module",
82	params=[True, False],
83	ids=["with_tool_parser", "without_tool_parser"],
84	)
85	def with_tool_parser(request) -> bool:
86	return request.param
87
88
89	@pytest.fixture(
90	scope="module",
91	params=[True],
92	ids=["exclude_tools_when_tool_choice_none"],
93	)
94	def exclude_tools_when_tool_choice_none(request) -> bool:
95	return request.param
96
97
98	@pytest.fixture(scope="module")
99	def default_server_args(
100	with_tool_parser: bool,
101	exclude_tools_when_tool_choice_none: bool,
102	):
103	args = [
104	# use half precision for speed and memory savings in CI environment
105	"--enforce-eager",
106	"--max-model-len",
107	"4096",
108	"--reasoning-parser",
109	"openai_gptoss",
110	"--gpu-memory-utilization",
111	"0.85",
112	]
113	if with_tool_parser:
114	args.extend(
115	[
116	"--tool-call-parser",
117	"openai",
118	"--enable-auto-tool-choice",
119	]
120	)
121	if exclude_tools_when_tool_choice_none:
122	args.append("--exclude-tools-when-tool-choice-none")
123	return args
124
125
126	@pytest.fixture(scope="class")
127	def gptoss_server(default_server_args: list[str]):
128	server_args = default_server_args + ["--attention-backend=TRITON_ATTN"]
129	with RemoteOpenAIServer(GPT_OSS_MODEL_NAME, server_args) as remote_server:
130	yield remote_server
131
132
133	@pytest.fixture(scope="class")
134	def gptoss_speculative_server(default_server_args: list[str]):
135	attention_backend = (
136	"TRITON_ATTN"
137	if not is_aiter_found_and_supported()
138	else "ROCM_AITER_UNIFIED_ATTN"
139	)
140	server_args = default_server_args + [
141	"--speculative-config",
142	f'{{"model": "{GPT_OSS_SPECULATOR_NAME}", '
143	f'"method": "eagle3", "num_speculative_tokens": 3}}',
144	f"--attention-backend={attention_backend}",
145	]
146	# gpt-oss requires AITER unified attention on ROCm
147	# TODO: Remove after fixing TRITON_ATTN issue on ROCm
148	# https://github.com/vllm-project/vllm/issues/32434
149	env_dict = None
150	if is_aiter_found_and_supported():
151	env_dict = {"VLLM_ROCM_USE_AITER": "1"}
152	with RemoteOpenAIServer(
153	GPT_OSS_MODEL_NAME, server_args, env_dict=env_dict, max_wait_seconds=480
154	) as remote_server:
155	yield remote_server
156
157
158	@pytest_asyncio.fixture
159	async def gptoss_client(gptoss_server):
160	async with gptoss_server.get_async_client() as async_client:
161	yield async_client
162
163
164	@pytest_asyncio.fixture
165	async def gptoss_speculative_client(gptoss_speculative_server):
166	async with gptoss_speculative_server.get_async_client() as async_client:
167	yield async_client
168
169
170	class TestGPTOSSChat:
171	@pytest.mark.asyncio
172	async def test_gpt_oss_chat_tool_call_streaming(
173	self, gptoss_client: OpenAI, with_tool_parser: bool
174	):
175	tools = [
176	{
177	"type": "function",
178	"function": {
179	"name": "get_current_weather",
180	"description": "Get the current weather in a given location",
181	"parameters": {
182	"type": "object",
183	"properties": {
184	"city": {"type": "string"},
185	"state": {"type": "string"},
186	"unit": {
187	"type": "string",
188	"enum": ["celsius", "fahrenheit"],
189	},
190	},
191	"required": ["city", "state", "unit"],
192	},
193	},
194	}
195	]
196
197	messages = [
198	{"role": "user", "content": "What is the weather in Dallas, TX?"},
199	]
200
201	stream = await gptoss_client.chat.completions.create(
202	model=GPT_OSS_MODEL_NAME,
203	messages=messages,
204	tools=tools if with_tool_parser else None,
205	stream=True,
206	)
207
208	name = None
209	args_buf = ""
210	content_buf = ""
211	async for chunk in stream:
212	delta = chunk.choices[0].delta
213	if delta.tool_calls:
214	tc = delta.tool_calls[0]
215	if tc.function and tc.function.name:
216	name = tc.function.name
217	if tc.function and tc.function.arguments:
218	args_buf += tc.function.arguments
219	if getattr(delta, "content", None):
220	content_buf += delta.content
221	if with_tool_parser:
222	assert name is not None
223	assert len(args_buf) > 0
224	else:
225	assert name is None
226	assert len(args_buf) == 0
227	assert len(content_buf) > 0
228
229	@pytest.mark.asyncio
230	async def test_gpt_oss_multi_turn_chat(
231	self, gptoss_client: OpenAI, with_tool_parser: bool
232	):
233	if not with_tool_parser:
234	pytest.skip("skip non-tool for multi-turn tests")
235	tools = [
236	{
237	"type": "function",
238	"function": {
239	"name": "get_current_weather",
240	"description": "Get the current weather in a given location",
241	"parameters": {
242	"type": "object",
243	"properties": {
244	"city": {"type": "string"},
245	"state": {"type": "string"},
246	"unit": {
247	"type": "string",
248	"enum": ["celsius", "fahrenheit"],
249	},
250	},
251	"required": ["city", "state", "unit"],
252	},
253	},
254	}
255	]
256
257	messages = [
258	{"role": "system", "content": "you are a helpful assistant"},
259	{
260	"role": "user",
261	"content": "What is the weather in Dallas, TX with celsius?",
262	},
263	]
264
265	first = await gptoss_client.chat.completions.create(
266	model=GPT_OSS_MODEL_NAME,
267	messages=messages,
268	tools=tools,
269	temperature=0.0,
270	)
271	first_msg = first.choices[0].message
272	assert first_msg.tool_calls is not None and len(first_msg.tool_calls) > 0
273	tc = first_msg.tool_calls[0]
274	assert tc.function is not None and tc.function.name == "get_current_weather"
275	args1 = tc.function.arguments
276	assert args1 is not None and len(args1) > 0
277	assert not first_msg.content
278
279	messages.append({"role": "assistant", "content": args1})
280	messages.append(
281	{"role": "user", "content": "Now convert to celsius and return JSON only"}
282	)
283
284	second = await gptoss_client.chat.completions.create(
285	model=GPT_OSS_MODEL_NAME,
286	messages=messages,
287	tools=tools,
288	temperature=0.0,
289	)
290	second_msg = second.choices[0].message
291	assert (second_msg.content is not None and len(second_msg.content) > 0) or (
292	second_msg.tool_calls is not None and len(second_msg.tool_calls) > 0
293	)
294
295	@pytest.mark.asyncio
296	async def test_gpt_oss_tool_message_array_content(
297	self, gptoss_client: OpenAI, with_tool_parser: bool
298	):
299	"""Test that tool messages support both string and array content formats."""
300	if not with_tool_parser:
301	pytest.skip("skip non-tool for array content tests")
302
303	tools = [
304	{
305	"type": "function",
306	"function": {
307	"name": "get_weather",
308	"description": "Get the current weather in a given location",
309	"parameters": {
310	"type": "object",
311	"properties": {
312	"city": {"type": "string"},
313	"state": {"type": "string"},
314	},
315	"required": ["city", "state"],
316	},
317	},
318	}
319	]
320
321	# Test 1: Tool message with string content
322	messages_string = [
323	{"role": "user", "content": "What's the weather in Paris?"},
324	{
325	"role": "assistant",
326	"tool_calls": [
327	{
328	"id": "call_123",
329	"type": "function",
330	"function": {
331	"name": "get_weather",
332	"arguments": '{"city": "Paris", "state": "TX"}',
333	},
334	}
335	],
336	},
337	{"role": "tool", "content": "The weather in Paris, TX is sunny, 22°C"},
338	]
339
340	response_string = await gptoss_client.chat.completions.create(
341	model=GPT_OSS_MODEL_NAME,
342	messages=messages_string,
343	tools=tools,
344	temperature=0.0,
345	)
346
347	assert response_string is not None
348	assert response_string.choices[0].message is not None
349
350	# Test 2: Tool message with array content
351	messages_array = [
352	{"role": "user", "content": "What's the weather in Dallas?"},
353	{
354	"role": "assistant",
355	"tool_calls": [
356	{
357	"id": "call_456",
358	"type": "function",
359	"function": {
360	"name": "get_weather",
361	"arguments": '{"city": "Dallas", "state": "TX"}',
362	},
363	}
364	],
365	},
366	{
367	"role": "tool",
368	"content": [
369	{"type": "text", "text": "f2e897a7-2705-4337-8193-2a8f57b81618"}
370	],
371	},
372	]
373
374	response_array = await gptoss_client.chat.completions.create(
375	model=GPT_OSS_MODEL_NAME,
376	messages=messages_array,
377	tools=tools,
378	temperature=0.0,
379	)
380
381	assert response_array is not None
382	assert response_array.choices[0].message is not None
383
384	# Test 3: Tool message with multiple array content items
385	messages_multi_array = [
386	{"role": "user", "content": "Search for information"},
387	{
388	"role": "assistant",
389	"tool_calls": [
390	{
391	"id": "call_789",
392	"type": "function",
393	"function": {
394	"name": "get_weather",
395	"arguments": '{"city": "Austin", "state": "TX"}',
396	},
397	}
398	],
399	},
400	{
401	"role": "tool",
402	"content": [
403	{"type": "text", "text": "Weather data: "},
404	{"type": "text", "text": "Austin, TX - Partly cloudy, 25°C"},
405	{"type": "text", "text": " with 60% humidity"},
406	],
407	},
408	]
409
410	response_multi_array = await gptoss_client.chat.completions.create(
411	model=GPT_OSS_MODEL_NAME,
412	messages=messages_multi_array,
413	tools=tools,
414	temperature=0.0,
415	)
416
417	assert response_multi_array is not None
418	assert response_multi_array.choices[0].message is not None
419
420	@pytest.mark.asyncio
421	async def test_gpt_oss_tool_choice_none(
422	self,
423	gptoss_client: OpenAI,
424	with_tool_parser: bool,
425	exclude_tools_when_tool_choice_none: bool,
426	):
427	if not (with_tool_parser and exclude_tools_when_tool_choice_none):
428	pytest.skip(
429	"skip tool_choice tests when non-tool or "
430	"--exclude-tools-when-tool-choice-none not set"
431	)
432
433	tools = [
434	{
435	"type": "function",
436	"function": {
437	"name": "get_current_weather",
438	"description": "Get the current weather in a given location",
439	"parameters": {
440	"type": "object",
441	"properties": {
442	"city": {"type": "string"},
443	"state": {"type": "string"},
444	"unit": {
445	"type": "string",
446	"enum": ["celsius", "fahrenheit"],
447	},
448	},
449	"required": ["city", "state", "unit"],
450	},
451	},
452	}
453	]
454
455	messages = [
456	{
457	"role": "user",
458	"content": "What's the temperature(in degrees Celsius) in Dallas?",
459	},
460	]
461
462	tool_choice_auto = await gptoss_client.chat.completions.create(
463	model=GPT_OSS_MODEL_NAME,
464	messages=messages,
465	tools=tools,
466	tool_choice="auto",
467	temperature=0.0,
468	)
469	msg = tool_choice_auto.choices[0].message
470	assert len(msg.tool_calls) == 1
471
472	tool_choice_none = await gptoss_client.chat.completions.create(
473	model=GPT_OSS_MODEL_NAME,
474	messages=messages,
475	tools=tools,
476	tool_choice="none",
477	temperature=0.0,
478	)
479
480	msg = tool_choice_none.choices[0].message
481	assert msg.tool_calls is None
482
483
484	class TestGPTOSSSpeculativeChat:
485	@pytest.mark.asyncio
486	async def test_gpt_oss_speculative_reasoning_leakage(
487	self,
488	gptoss_speculative_client: OpenAI,
489	with_tool_parser: bool,
490	):
491	if not with_tool_parser:
492	pytest.skip("skip non-tool for array content tests")
493
494	messages = [
495	{"role": "user", "content": "Calculate 2+2. Return the answer 4 only."},
496	]
497
498	stream = await gptoss_speculative_client.chat.completions.create(
499	model=GPT_OSS_MODEL_NAME,
500	messages=messages,
501	stream=True,
502	temperature=0.0,
503	)
504
505	content = ""
506	reasoning = ""
507	async for chunk in stream:
508	delta = chunk.choices[0].delta
509	if delta.content:
510	content += delta.content
511
512	chunk_reasoning = getattr(delta, "reasoning", None)
513	if chunk_reasoning:
514	reasoning += delta.reasoning
515
516	assert len(reasoning) > 0, "No reasoning was generated."
517	assert content.strip() == "4"
518
519
520	MODEL_NAME = "openai-community/gpt2"
521	MODEL_NAME_SHORT = "gpt2"
522	CHAT_TEMPLATE = "Dummy chat template for testing {}"
523	BASE_MODEL_PATHS = [
524	BaseModelPath(name=MODEL_NAME, model_path=MODEL_NAME),
525	BaseModelPath(name=MODEL_NAME_SHORT, model_path=MODEL_NAME_SHORT),
526	]
527
528
529	@dataclass
530	class MockHFConfig:
531	model_type: str = "any"
532
533
534	@dataclass
535	class MockModelConfig:
536	task = "generate"
537	runner_type = "generate"
538	model = MODEL_NAME
539	tokenizer = MODEL_NAME
540	trust_remote_code = False
541	tokenizer_mode = "auto"
542	max_model_len = 100
543	revision = None
544	code_revision = None
545	tokenizer_revision = None
546	multimodal_config = MultiModalConfig()
547	hf_config = MockHFConfig()
548	hf_text_config = MockHFConfig()
549	logits_processors: list[str] \| None = None
550	diff_sampling_param: dict \| None = None
551	allowed_local_media_path: str = ""
552	allowed_media_domains: list[str] \| None = None
553	encoder_config = None
554	generation_config: str = "auto"
555	override_generation_config: dict[str, Any] = field(default_factory=dict)
556	media_io_kwargs: dict[str, dict[str, Any]] = field(default_factory=dict)
557	skip_tokenizer_init: bool = False
558	is_encoder_decoder: bool = False
559	is_multimodal_model: bool = False
560	supports_multimodal_inputs: bool = False
561	renderer_num_workers: int = 1
562	enable_prompt_embeds: bool = False
563
564	def get_diff_sampling_param(self):
565	return self.diff_sampling_param or {}
566
567	def get_vocab_size(self) -> int:
568	return 50257
569
570
571	@dataclass
572	class MockParallelConfig:
573	_api_process_rank: int = 0
574
575
576	@dataclass
577	class MockVllmConfig:
578	model_config: MockModelConfig
579	parallel_config: MockParallelConfig
580
581
582	def _build_renderer(model_config: MockModelConfig):
583	return HfRenderer(
584	MockVllmConfig(model_config, parallel_config=MockParallelConfig()),
585	cached_tokenizer_from_config(model_config),
586	)
587
588
589	def _build_online_renderer(
590	engine, model_registry: OpenAIModelRegistry
591	) -> OnlineRenderer:
592	return OnlineRenderer(
593	model_config=engine.model_config,
594	renderer=engine.renderer,
595	request_logger=None,
596	chat_template=CHAT_TEMPLATE,
597	chat_template_content_format="auto",
598	)
599
600
601	def _build_mock_engine() -> MagicMock:
602	mock_engine = MagicMock(spec=AsyncLLM)
603	mock_engine.errored = False
604	mock_engine.model_config = MockModelConfig()
605	mock_engine.input_processor = MagicMock()
606	mock_engine.renderer = _build_renderer(mock_engine.model_config)
607	return mock_engine
608
609
610	def _build_serving_chat(
611	engine: AsyncLLM,
612	*,
613	reasoning_parser: str = "",
614	tool_parser: str \| None = None,
615	enable_auto_tools: bool = False,
616	) -> OpenAIServingChat:
617	models = OpenAIServingModels(
618	engine_client=engine,
619	base_model_paths=BASE_MODEL_PATHS,
620	)
621	online_renderer = _build_online_renderer(engine, models.registry)
622
623	serving_chat = OpenAIServingChat(
624	engine,
625	models,
626	response_role="assistant",
627	online_renderer=online_renderer,
628	chat_template=CHAT_TEMPLATE,
629	chat_template_content_format="auto",
630	request_logger=None,
631	reasoning_parser=reasoning_parser,
632	tool_parser=tool_parser,
633	enable_auto_tools=enable_auto_tools,
634	)
635
636	return serving_chat
637
638
639	def _build_minimal_metrics_serving_chat(
640	enable_per_request_metrics: bool,
641	enable_force_include_usage: bool = False,
642	) -> OpenAIServingChat:
643	serving = OpenAIServingChat.__new__(OpenAIServingChat)
644	serving.response_role = "assistant"
645	serving.parser_cls = None
646	serving.enable_auto_tools = False
647	serving._include_reasoning_tokens_details = False
648	serving.enable_prompt_tokens_details = False
649	serving.enable_log_outputs = False
650	serving.enable_log_deltas = False
651	serving.enable_force_include_usage = enable_force_include_usage
652	serving.request_logger = None
653	serving.system_fingerprint = None
654	serving.enable_per_request_metrics = enable_per_request_metrics
655	return serving
656
657
658	def _make_metrics_request_output(
659	metrics: RequestStateStats \| None = _PER_REQUEST_STATS,
660	token_ids: tuple[int, ...] = (100, 101),
661	) -> RequestOutput:
662	return RequestOutput(
663	request_id="test-id",
664	prompt="Test prompt",
665	prompt_token_ids=[1, 2, 3],
666	prompt_logprobs=None,
667	outputs=[
668	CompletionOutput(
669	index=0,
670	text="Hello",
671	token_ids=list(token_ids),
672	cumulative_logprob=None,
673	logprobs=None,
674	finish_reason="stop",
675	)
676	],
677	finished=True,
678	metrics=metrics,
679	)
680
681
682	async def _single_request_output(
683	request_output: RequestOutput,
684	) -> AsyncIterator[RequestOutput]:
685	yield request_output
686
687
688	async def _stream_request_outputs(
689	*request_outputs: RequestOutput,
690	) -> AsyncIterator[RequestOutput]:
691	for request_output in request_outputs:
692	yield request_output
693
694
695	async def _collect_metrics_stream_chunks(
696	serving: OpenAIServingChat,
697	request: ChatCompletionRequest,
698	) -> list[dict[str, Any]]:
699	chunks: list[dict[str, Any]] = []
700	async for line in serving.chat_completion_stream_generator(
701	request,
702	_single_request_output(_make_metrics_request_output()),
703	"chatcmpl-test-id",
704	"test-model",
705	conversation=[{"role": "user", "content": "Test"}],
706	tokenizer=MagicMock(),
707	request_metadata=RequestResponseMetadata(request_id="chatcmpl-test-id"),
708	):
709	line = line.strip()
710	if not line.startswith("data: "):
711	continue
712	payload = line[len("data: ") :]
713	if payload != "[DONE]":
714	chunks.append(json.loads(payload))
715	return chunks
716
717
718	def test_build_per_request_timing_metrics_valid_timestamps():
719	metrics = build_per_request_timing_metrics(
720	_PER_REQUEST_STATS, num_generation_tokens=10
721	)
722
723	assert metrics.time_to_first_token_ms == pytest.approx(500.0)
724	assert metrics.generation_time_ms == pytest.approx(1000.0)
725	assert metrics.queue_time_ms == pytest.approx(500.0)
726	assert metrics.mean_itl_ms == pytest.approx(1000.0 / 9, rel=1e-4)
727	assert metrics.tokens_per_second == pytest.approx(10.0 / 1.5, rel=1e-4)
728
729
730	@pytest.mark.asyncio
731	async def test_chat_per_request_metrics_follow_server_flag():
732	request = ChatCompletionRequest(
733	model="test-model",
734	messages=[{"role": "user", "content": "Test prompt"}],
735	max_tokens=10,
736	stream=False,
737	)
738	request_output = _make_metrics_request_output()
739
740	disabled_serving = _build_minimal_metrics_serving_chat(
741	enable_per_request_metrics=False
742	)
743	disabled_response = await disabled_serving.chat_completion_full_generator(
744	request,
745	_single_request_output(request_output),
746	"chatcmpl-test-id",
747	"test-model",
748	conversation=[{"role": "user", "content": "Test"}],
749	tokenizer=MagicMock(),
750	request_metadata=RequestResponseMetadata(request_id="chatcmpl-test-id"),
751	)
752	assert disabled_response.metrics is None
753	details = disabled_response.usage.completion_tokens_details
754	assert details is None or details.reasoning_tokens == 0
755
756	enabled_serving = _build_minimal_metrics_serving_chat(
757	enable_per_request_metrics=True
758	)
759	enabled_response = await enabled_serving.chat_completion_full_generator(
760	request,
761	_single_request_output(request_output),
762	"chatcmpl-test-id",
763	"test-model",
764	conversation=[{"role": "user", "content": "Test"}],
765	tokenizer=MagicMock(),
766	request_metadata=RequestResponseMetadata(request_id="chatcmpl-test-id"),
767	)
768	assert enabled_response.metrics is not None
769	assert enabled_response.metrics.time_to_first_token_ms == pytest.approx(500.0)
770
771
772	@pytest.mark.asyncio
773	async def test_chat_per_request_metrics_suppressed_for_n_greater_than_one():
774	serving = _build_minimal_metrics_serving_chat(enable_per_request_metrics=True)
775	response = await serving.chat_completion_full_generator(
776	ChatCompletionRequest(
777	model="test-model",
778	messages=[{"role": "user", "content": "Test prompt"}],
779	max_tokens=10,
780	stream=False,
781	n=2,
782	),
783	_single_request_output(_make_metrics_request_output()),
784	"chatcmpl-test-id",
785	"test-model",
786	conversation=[{"role": "user", "content": "Test"}],
787	tokenizer=MagicMock(),
788	request_metadata=RequestResponseMetadata(request_id="chatcmpl-test-id"),
789	)
790	assert response.metrics is None
791
792
793	@pytest.mark.asyncio
794	async def test_chat_full_generator_gives_each_choice_parser_the_prompt():
795	prompts: list[list[int] \| None] = []
796
797	class RecordingParser:
798	tool_parser_cls = None
799
800	def __init__(self, args, *kwargs):
801	self.prompt_token_ids = None
802
803	def set_prompt_token_ids(self, prompt_token_ids):
804	self.prompt_token_ids = list(prompt_token_ids)
805
806	def parse(self, model_output, request, **kwargs):
807	prompts.append(self.prompt_token_ids)
808	return None, model_output, None
809
810	def count_reasoning_tokens(self, token_ids):
811	return 0
812
813	serving = _build_minimal_metrics_serving_chat(enable_per_request_metrics=False)
814	serving.parser_cls = RecordingParser
815	serving.model_config = None
816	serving.chat_template = None
817	serving.chat_template_content_format = "auto"
818	serving.default_chat_template_kwargs = {}
819	request_output = _make_metrics_request_output()
820	request_output.prompt_token_ids = [7, 8]
821	request_output.outputs.append(
822	CompletionOutput(
823	index=1,
824	text="Hi",
825	token_ids=[102],
826	cumulative_logprob=None,
827	logprobs=None,
828	finish_reason="stop",
829	)
830	)
831
832	await serving.chat_completion_full_generator(
833	ChatCompletionRequest(
834	model="test-model",
835	messages=[{"role": "user", "content": "Test prompt"}],
836	max_tokens=10,
837	n=2,
838	),
839	_single_request_output(request_output),
840	"chatcmpl-test-id",
841	"test-model",
842	conversation=[{"role": "user", "content": "Test"}],
843	tokenizer=MagicMock(),
844	request_metadata=RequestResponseMetadata(request_id="chatcmpl-test-id"),
845	parser=RecordingParser(),
846	)
847
848	assert prompts == [[7, 8], [7, 8]]
849
850
851	@pytest.mark.asyncio
852	async def test_chat_streaming_metrics_ride_on_usage_chunk():
853	serving = _build_minimal_metrics_serving_chat(enable_per_request_metrics=True)
854	chunks = await _collect_metrics_stream_chunks(
855	serving,
856	ChatCompletionRequest(
857	model="test-model",
858	messages=[{"role": "user", "content": "Test prompt"}],
859	max_tokens=10,
860	stream=True,
861	stream_options={"include_usage": True},
862	),
863	)
864
865	usage_chunks = [chunk for chunk in chunks if chunk.get("usage")]
866	assert usage_chunks
867	assert usage_chunks[-1]["metrics"]["time_to_first_token_ms"] == pytest.approx(500.0)
868
869
870	@pytest.mark.asyncio
871	@pytest.mark.parametrize("n", [1, 2])
872	@pytest.mark.parametrize("include_reasoning", [True, False])
873	@pytest.mark.parametrize("include_details", [True, False])
874	@pytest.mark.parametrize("separate_finish", [True, False])
875	@pytest.mark.parametrize(
876	"stream_options, force_usage, include_usage, continuous_usage",
877	[
878	(None, False, False, False),
879	({"include_usage": True}, False, True, False),
880	({"include_usage": True, "continuous_usage_stats": True}, False, True, True),
881	({"include_usage": False, "continuous_usage_stats": True}, False, False, False),
882	(None, True, True, True),
883	],
884	)
885	async def test_streaming_reasoning_usage_counts_across_deltas(
886	n,
887	include_reasoning,
888	include_details,
889	separate_finish,
890	stream_options,
891	force_usage,
892	include_usage,
893	continuous_usage,
894	):
895	"""Final usage needs one recount per choice, not per streamed delta."""
896	serving = _build_minimal_metrics_serving_chat(
897	enable_per_request_metrics=False, enable_force_include_usage=force_usage
898	)
899	serving._include_reasoning_tokens_details = include_details
900	serving.model_config = None
901
902	parsers = [MagicMock() for _ in range(n)]
903	request_outputs = []
904	num_steps = 3 if separate_finish else 2
905	for parser in parsers:
906	parser.parse_delta.side_effect = [
907	DeltaMessage(reasoning="reasoning") if include_reasoning else None,
908	DeltaMessage(content="answer"),
909	DeltaMessage(),
910	]
911	parser.count_reasoning_tokens.side_effect = lambda token_ids: sum(
912	token_id == 20 for token_id in token_ids
913	)
914	serving.parser_cls = MagicMock(side_effect=parsers)
915	for step in range(num_steps):
916	for i in reversed(range(n)):
917	tokens = ((10,) + (20,) * (i + 1), (11, 20, 30), ())[step]
918	result = _make_metrics_request_output(metrics=None, token_ids=tokens)
919	result.outputs[0].index = i
920	result.outputs[0].text = ("<think>reasoning", "</think>answer", "")[step]
921	finished = step == num_steps - 1
922	result.outputs[0].finish_reason = "stop" if finished else None
923	result.finished = finished and i == 0
924	request_outputs.append(result)
925
926	request = ChatCompletionRequest(
927	model="test-model",
928	messages=[{"role": "user", "content": "Test prompt"}],
929	max_tokens=10,
930	stream=True,
931	n=n,
932	include_reasoning=include_reasoning,
933	return_token_ids=True,
934	stream_options=stream_options,
935	)
936	metadata = RequestResponseMetadata(request_id="chatcmpl-test-id")
937	chunks: list[dict[str, Any]] = []
938	async for line in serving.chat_completion_stream_generator(
939	request,
940	_stream_request_outputs(*request_outputs),
941	"chatcmpl-test-id",
942	"test-model",
943	conversation=[{"role": "user", "content": "Test"}],
944	tokenizer=MagicMock(),
945	request_metadata=metadata,
946	):
947	payload = line.removeprefix("data: ").strip()
948	if payload != "[DONE]":
949	chunks.append(json.loads(payload))
950
951	assert all("error" not in chunk for chunk in chunks)
952	expected_count = sum(i + 2 for i in range(n))
953	expected_details = {"reasoning_tokens": expected_count} if include_details else None
954	assert metadata.final_usage_info is not None
955	assert metadata.final_usage_info.model_dump()["completion_tokens_details"] == (
956	expected_details
957	)
958	final_chunks = [chunk for chunk in chunks if not chunk["choices"]]
959	assert len(final_chunks) == int(include_usage)
960	if include_usage:
961	assert final_chunks[0]["usage"].get("completion_tokens_details") == (
962	expected_details
963	)
964	for i, parser in enumerate(parsers):
965	expected_calls = (
966	(num_steps if continuous_usage else 1) if include_details else 0
967	)
968	assert parser.count_reasoning_tokens.call_count == expected_calls
969	if include_details:
970	assert parser.count_reasoning_tokens.call_args.args == (
971	(10,) + (20,) * (i + 1) + (11, 20, 30),
972	)
973	choices = [
974	chunk
975	for chunk in chunks
976	if chunk["choices"] and chunk["choices"][0]["index"] == i
977	]
978	assert (
979	"".join(c["choices"][0]["delta"].get("content") or "" for c in choices)
980	== "answer"
981	)
982	if not include_reasoning:
983	assert all(c["choices"][0].get("token_ids") is None for c in choices)
984	if continuous_usage and include_details:
985	counts = [
986	c["usage"]["completion_tokens_details"]["reasoning_tokens"]
987	for c in choices
988	]
989	expected_counts = [0, i + 1] if include_reasoning else [0]
990	assert counts == expected_counts + [i + 2] * (num_steps - 1)
991
992
993	def _reasoning_lifecycle_output(
994	index: int = 0,
995	tokens: tuple[int, ...] = (20, 30),
996	text: str = "answer",
997	finish: str \| None = None,
998	) -> RequestOutput:
999	result = _make_metrics_request_output(None, tokens)
1000	result.outputs[0].index = index
1001	result.outputs[0].text = text
1002	result.outputs[0].finish_reason = finish
1003	result.finished = finish is not None
1004	return result
1005
1006
1007	_EMPTY_OUTPUTS = _reasoning_lifecycle_output(tokens=(), text="")
1008	_EMPTY_OUTPUTS.outputs = []
1009
1010	_REASONING_LIFECYCLE_CASES = [
1011	("empty-generator", 1, [], 0, 0),
1012	("empty-outputs", 1, [_EMPTY_OUTPUTS], 0, 0),
1013	(
1014	"empty-finish",
1015	1,
1016	[_reasoning_lifecycle_output(tokens=(), text="", finish="stop")],
1017	0,
1018	0,
1019	),
1020	(
1021	"empty-prefill",
1022	1,
1023	[
1024	_reasoning_lifecycle_output(tokens=(), text=""),
1025	_reasoning_lifecycle_output(finish="stop"),
1026	],
1027	2,
1028	1,
1029	),
1030	(
1031	"one-empty-choice",
1032	2,
1033	[_reasoning_lifecycle_output(index=1, finish="stop")],
1034	2,
1035	1,
1036	),
1037	(
1038	"uneven-finish",
1039	2,
1040	[
1041	_reasoning_lifecycle_output(index=1, finish="stop"),
1042	_reasoning_lifecycle_output(),
1043	_reasoning_lifecycle_output(tokens=(20,), finish="stop"),
1044	],
1045	5,
1046	3,
1047	),
1048	(
1049	"empty-final-delta",
1050	1,
1051	[
1052	_reasoning_lifecycle_output(),
1053	_reasoning_lifecycle_output(tokens=(), text="", finish="stop"),
1054	],
1055	2,
1056	1,
1057	),
1058	(
1059	"ignore-finished-choice",
1060	2,
1061	[
1062	_reasoning_lifecycle_output(index=1, finish="stop"),
1063	_reasoning_lifecycle_output(index=1, tokens=(20, 20, 20)),
1064	_reasoning_lifecycle_output(finish="stop"),
1065	],
1066	4,
1067	2,
1068	),
1069	("exception-before", 1, [RuntimeError("fixture engine error")], None, None),
1070	(
1071	"exception-after",
1072	1,
1073	[_reasoning_lifecycle_output(), RuntimeError("fixture engine error")],
1074	None,
1075	None,
1076	),
1077	(
1078	"generation-error",
1079	1,
1080	[
1081	_reasoning_lifecycle_output(),
1082	_reasoning_lifecycle_output(finish="error"),
1083	],
1084	None,
1085	None,
1086	),
1087	(
1088	"cancel-after",
1089	1,
1090	[_reasoning_lifecycle_output(), asyncio.CancelledError()],
1091	None,
1092	None,
1093	),
1094	(
1095	"close-after",
1096	1,
1097	[
1098	_reasoning_lifecycle_output(),
1099	_reasoning_lifecycle_output(finish="stop"),
1100	],
1101	None,
1102	None,
1103	),
1104	]
1105
1106
1107	@pytest.mark.asyncio
1108	@pytest.mark.parametrize("usage_mode", ["none", "final", "continuous"])
1109	@pytest.mark.parametrize(
1110	"name, n, outputs, expected_tokens, expected_reasoning",
1111	_REASONING_LIFECYCLE_CASES,
1112	ids=[case[0] for case in _REASONING_LIFECYCLE_CASES],
1113	)
1114	async def test_streaming_reasoning_usage_lifecycle_edges(
1115	usage_mode,
1116	name,
1117	n,
1118	outputs,
1119	expected_tokens,
1120	expected_reasoning,
1121	):
1122	"""Cancel, errors, and empty choices still match baseline stream usage."""
1123	serving = _build_minimal_metrics_serving_chat(enable_per_request_metrics=False)
1124	serving.model_config = None
1125	serving._include_reasoning_tokens_details = True
1126	parsers = [MagicMock() for _ in range(n)]
1127	for parser in parsers:
1128	parser.parse_delta.side_effect = lambda **kwargs: DeltaMessage(
1129	content=kwargs["delta_text"]
1130	)
1131	parser.count_reasoning_tokens.side_effect = lambda ids: ids.count(20)
1132	serving.parser_cls = MagicMock(side_effect=parsers)
1133	stream_options = (
1134	None
1135	if usage_mode == "none"
1136	else {
1137	"include_usage": True,
1138	"continuous_usage_stats": usage_mode == "continuous",
1139	}
1140	)
1141	request = ChatCompletionRequest(
1142	model="test-model",
1143	messages=[{"role": "user", "content": "Test prompt"}],
1144	stream=True,
1145	n=n,
1146	return_token_ids=True,
1147	stream_options=stream_options,
1148	)
1149	metadata = RequestResponseMetadata(request_id="chatcmpl-lifecycle")
1150
1151	async def source():
1152	for result in outputs:
1153	if isinstance(result, BaseException):
1154	raise result
1155	yield result
1156
1157	iterator = source()
1158	stream = serving.chat_completion_stream_generator(
1159	request,
1160	iterator,
1161	"chatcmpl-lifecycle",
1162	"test-model",
1163	conversation=[{"role": "user", "content": "Test"}],
1164	tokenizer=MagicMock(),
1165	request_metadata=metadata,
1166	)
1167	chunks: list[object] = []
1168	outcome = "exhausted"
1169	try:
1170	async for line in stream:
1171	raw = line.removeprefix("data: ").strip()
1172	if raw == "[DONE]":
1173	chunks.append(raw)
1174	continue
1175	item = json.loads(raw)
1176	chunks.append(item)
1177	if name == "close-after" and any(
1178	choice["delta"].get("content") for choice in item.get("choices", [])
1179	):
1180	await stream.aclose()
1181	outcome = "closed"
1182	break
1183	except asyncio.CancelledError:
1184	outcome = "cancelled"
1185	finally:
1186	await stream.aclose()
1187	await iterator.aclose()
1188
1189	error_chunks = [c for c in chunks if isinstance(c, dict) and "error" in c]
1190	call_count = sum(parser.count_reasoning_tokens.call_count for parser in parsers)
1191	if expected_tokens is not None:
1192	assert outcome == "exhausted" and chunks[-1] == "[DONE]" and not error_chunks
1193	assert metadata.final_usage_info is not None
1194	assert metadata.final_usage_info.completion_tokens == expected_tokens
1195	assert (
1196	metadata.final_usage_info.completion_tokens_details.reasoning_tokens
1197	== expected_reasoning
1198	)
1199	nonempty_choices = sum(
1200	bool(parser.count_reasoning_tokens.call_args) for parser in parsers
1201	)
1202	if usage_mode == "continuous":
1203	assert call_count >= nonempty_choices
1204	else:
1205	assert call_count == nonempty_choices
1206	return
1207
1208	assert metadata.final_usage_info is None
1209	if name in {"cancel-after", "close-after"}:
1210	assert outcome == ("cancelled" if name == "cancel-after" else "closed")
1211	assert "[DONE]" not in chunks and not error_chunks
1212	else:
1213	assert outcome == "exhausted" and chunks[-1] == "[DONE]"
1214	assert len(error_chunks) == 1
1215	if usage_mode != "continuous" or name == "exception-before":
1216	assert call_count == 0
1217	else:
1218	assert call_count >= 1
1219
1220
1221	@pytest.mark.asyncio
1222	@pytest.mark.parametrize(
1223	("include_reasoning", "expected_tokens"),
1224	[(True, ["<tool_call>", "answer"]), (False, [])],
1225	)
1226	async def test_streaming_logprobs_kept_when_parser_suppresses_delta(
1227	include_reasoning: bool, expected_tokens: list[str]
1228	):
1229	"""Logprobs of tokens the parser swallows (e.g. <tool_call>) must still
1230	be streamed, unless per-token metadata is hidden with the reasoning."""
1231	serving = _build_minimal_metrics_serving_chat(enable_per_request_metrics=False)
1232	serving.model_config = None
1233	serving.return_tokens_as_token_ids = False
1234
1235	parser = MagicMock()
1236	parser.parse_delta.side_effect = [None, DeltaMessage(content="answer")]
1237	parser.count_reasoning_tokens.return_value = 0
1238	serving.parser_cls = MagicMock(return_value=parser)
1239
1240	def make_output(token_id: int, text: str, finish_reason: str \| None):
1241	request_output = _make_metrics_request_output(
1242	metrics=None, token_ids=(token_id,)
1243	)
1244	completion = request_output.outputs[0]
1245	completion.text = text
1246	completion.finish_reason = finish_reason
1247	completion.logprobs = [
1248	{token_id: Logprob(logprob=-0.5, rank=1, decoded_token=text)}
1249	]
1250	return request_output
1251
1252	request = ChatCompletionRequest(
1253	model="test-model",
1254	messages=[{"role": "user", "content": "Test prompt"}],
1255	max_tokens=10,
1256	stream=True,
1257	logprobs=True,
1258	include_reasoning=include_reasoning,
1259	)
1260	chunks: list[dict[str, Any]] = []
1261	async for line in serving.chat_completion_stream_generator(
1262	request,
1263	_stream_request_outputs(
1264	make_output(10, "<tool_call>", None), make_output(11, "answer", "stop")
1265	),
1266	"chatcmpl-test-id",
1267	"test-model",
1268	conversation=[{"role": "user", "content": "Test"}],
1269	tokenizer=MagicMock(),
1270	request_metadata=RequestResponseMetadata(request_id="chatcmpl-test-id"),
1271	):
1272	payload = line.removeprefix("data: ").strip()
1273	if payload != "[DONE]":
1274	chunks.append(json.loads(payload))
1275
1276	streamed_tokens = [
1277	entry["token"]
1278	for chunk in chunks
1279	for choice in chunk["choices"]
1280	if choice.get("logprobs")
1281	for entry in choice["logprobs"]["content"]
1282	]
1283	assert streamed_tokens == expected_tokens
1284
1285
1286	@dataclass
1287	class MockEngine:
1288	model_config: MockModelConfig = field(default_factory=MockModelConfig)
1289	input_processor: MagicMock = field(default_factory=MagicMock)
1290	renderer: MagicMock = field(default_factory=MagicMock)
1291	errored: bool = False
1292
1293	def check_admission(self, n: int = 1, request_id: str \| None = None) -> None:
1294	pass
1295
1296
1297	async def _async_serving_chat_init():
1298	engine = MockEngine()
1299
1300	models = OpenAIServingModels(engine, BASE_MODEL_PATHS)
1301	online_renderer = _build_online_renderer(engine, models.registry)
1302
1303	serving_completion = OpenAIServingChat(
1304	engine,
1305	models,
1306	response_role="assistant",
1307	online_renderer=online_renderer,
1308	chat_template=CHAT_TEMPLATE,
1309	chat_template_content_format="auto",
1310	request_logger=None,
1311	)
1312	return serving_completion
1313
1314
1315	def test_async_serving_chat_init():
1316	serving_completion = asyncio.run(_async_serving_chat_init())
1317	assert serving_completion.chat_template == CHAT_TEMPLATE
1318	assert serving_completion._include_reasoning_tokens_details is False
1319
1320
1321	def test_mm_prompt_tokens_details():
1322	# Text-only input has no multimodal placeholders.
1323	assert _get_mm_token_counts({"type": "tokens"}) == {}
1324
1325	# Per-modality counts sum each modality's placeholder ranges.
1326	counts = _get_mm_token_counts(
1327	{
1328	"mm_placeholders": {
1329	"image": [
1330	PlaceholderRange(offset=0, length=576),
1331	PlaceholderRange(offset=600, length=24),
1332	],
1333	"video": [PlaceholderRange(offset=700, length=1200)],
1334	}
1335	}
1336	)
1337	assert counts == {"image": 600, "video": 1200}
1338
1339	# Gated off, or nothing to report -> no details.
1340	assert _make_prompt_tokens_details(False, 5, 0, counts) is None
1341	assert _make_prompt_tokens_details(True, None, None, None) is None
1342
1343	# Zero cached_tokens is still reported (not None), matching the cached-only
1344	# behavior; multimodal counts ride alongside even when cached_tokens is None.
1345	details = _make_prompt_tokens_details(True, 0, 0, None)
1346	assert details.cached_tokens == 0
1347	assert details.created_cache_tokens == 0
1348	assert details.multimodal_tokens is None
1349	details = _make_prompt_tokens_details(True, None, None, counts)
1350	assert details.cached_tokens is None
1351	assert details.created_cache_tokens is None
1352	assert details.multimodal_tokens == {"image": 600, "video": 1200}
1353	details = _make_prompt_tokens_details(True, 3, 0, counts)
1354	assert details.cached_tokens == 3
1355	assert details.created_cache_tokens == 0
1356	assert details.multimodal_tokens == {"image": 600, "video": 1200}
1357
1358
1359	def test_completion_tokens_details():
1360	assert _make_completion_tokens_details(7).reasoning_tokens == 7
1361
1362
1363	@pytest.mark.asyncio
1364	async def test_serving_chat_returns_correct_model_name():
1365	mock_engine = MagicMock(spec=AsyncLLM)
1366	mock_engine.errored = False
1367	mock_engine.model_config = MockModelConfig()
1368	mock_engine.input_processor = MagicMock()
1369	mock_engine.renderer = _build_renderer(mock_engine.model_config)
1370
1371	serving_chat = _build_serving_chat(mock_engine)
1372	messages = [{"role": "user", "content": "what is 1+1?"}]
1373
1374	async def return_model_name(args, *kwargs):
1375	return args[3]
1376
1377	serving_chat.chat_completion_full_generator = return_model_name
1378
1379	# Test that full name is returned when short name is requested
1380	req = ChatCompletionRequest(model=MODEL_NAME_SHORT, messages=messages)
1381	assert await serving_chat.create_chat_completion(req) == MODEL_NAME
1382
1383	# Test that full name is returned when empty string is specified
1384	req = ChatCompletionRequest(model="", messages=messages)
1385	assert await serving_chat.create_chat_completion(req) == MODEL_NAME
1386
1387	# Test that full name is returned when no model is specified
1388	req = ChatCompletionRequest(messages=messages)
1389	assert await serving_chat.create_chat_completion(req) == MODEL_NAME
1390
1391
1392	@pytest.mark.asyncio
1393	@pytest.mark.parametrize("stream", [True, False])
1394	async def test_admission_rejection_escapes_before_response_starts(stream):
1395	"""Overload rejections must propagate out of create_chat_completion.
1396
1397	For streaming this is the only chance to return a real HTTP status: once
1398	StreamingResponse is constructed the 200 has already been sent and the
1399	rejection would degrade into an in-band SSE error chunk.
1400	"""
1401	mock_engine = MagicMock(spec=AsyncLLM)
1402	mock_engine.errored = False
1403	mock_engine.model_config = MockModelConfig()
1404	mock_engine.input_processor = MagicMock()
1405	mock_engine.renderer = _build_renderer(mock_engine.model_config)
1406	mock_engine.check_admission.side_effect = QueueOverflowError()
1407
1408	serving_chat = _build_serving_chat(mock_engine)
1409	req = ChatCompletionRequest(
1410	model=MODEL_NAME,
1411	messages=[{"role": "user", "content": "what is 1+1?"}],
1412	stream=stream,
1413	)
1414
1415	with pytest.raises(QueueOverflowError):
1416	await serving_chat.create_chat_completion(req)
1417
1418	mock_engine.generate.assert_not_called()
1419
1420
1421	@pytest.mark.asyncio
1422	async def test_serving_chat_should_set_correct_max_tokens():
1423	mock_engine = MagicMock(spec=AsyncLLM)
1424	mock_engine.errored = False
1425	mock_engine.model_config = MockModelConfig()
1426	mock_engine.input_processor = MagicMock()
1427	mock_engine.renderer = _build_renderer(mock_engine.model_config)
1428
1429	serving_chat = _build_serving_chat(mock_engine)
1430
1431	req = ChatCompletionRequest(
1432	model=MODEL_NAME,
1433	messages=[{"role": "user", "content": "what is 1+1?"}],
1434	)
1435
1436	with suppress(Exception):
1437	await serving_chat.create_chat_completion(req)
1438
1439	assert mock_engine.generate.call_args.args[1].max_tokens == 93
1440
1441	req.max_tokens = 10
1442	with suppress(Exception):
1443	await serving_chat.create_chat_completion(req)
1444
1445	assert mock_engine.generate.call_args.args[1].max_tokens == 10
1446
1447	# Model author's generation_config.json sets max_tokens (auto, no override)
1448	# — should act as fallback only, not ceiling
1449	mock_model_config = MockModelConfig()
1450	mock_model_config.diff_sampling_param = {"max_tokens": 10}
1451
1452	# Reinitialize the engine with new settings
1453	mock_engine = MagicMock(spec=AsyncLLM)
1454	mock_engine.errored = False
1455	mock_engine.model_config = mock_model_config
1456	mock_engine.input_processor = MagicMock()
1457	mock_engine.renderer = _build_renderer(mock_engine.model_config)
1458
1459	# Initialize the serving chat
1460	serving_chat = _build_serving_chat(mock_engine)
1461
1462	# Test Case 1: No max_tokens specified in request
1463	req = ChatCompletionRequest(
1464	model=MODEL_NAME,
1465	messages=[{"role": "user", "content": "what is 1+1?"}],
1466	)
1467
1468	with suppress(Exception):
1469	await serving_chat.create_chat_completion(req)
1470
1471	assert mock_engine.generate.call_args.args[1].max_tokens == 10
1472
1473	# Test Case 2: Request's max_tokens set higher than generation_config
1474	# default so request-provided max_tokens takes precedence
1475	req.max_tokens = 15
1476
1477	with suppress(Exception):
1478	await serving_chat.create_chat_completion(req)
1479
1480	assert mock_engine.generate.call_args.args[1].max_tokens == 15
1481
1482	# Test Case 3: Request's max_tokens set lower than server accepts
1483	req.max_tokens = 5
1484
1485	with suppress(Exception):
1486	await serving_chat.create_chat_completion(req)
1487
1488	assert mock_engine.generate.call_args.args[1].max_tokens == 5
1489
1490	# User explicitly sets max_tokens via --override-generation-config
1491	# — should act as a ceiling
1492	mock_model_config = MockModelConfig()
1493	mock_model_config.diff_sampling_param = {"max_tokens": 10}
1494	mock_model_config.override_generation_config = {"max_new_tokens": 10}
1495
1496	mock_engine = MagicMock(spec=AsyncLLM)
1497	mock_engine.errored = False
1498	mock_engine.model_config = mock_model_config
1499	mock_engine.input_processor = MagicMock()
1500	mock_engine.renderer = _build_renderer(mock_engine.model_config)
1501
1502	serving_chat = _build_serving_chat(mock_engine)
1503
1504	# Test Case 3.1: No max_tokens — uses override as default
1505	req = ChatCompletionRequest(
1506	model=MODEL_NAME,
1507	messages=[{"role": "user", "content": "what is 1+1?"}],
1508	)
1509
1510	with suppress(Exception):
1511	await serving_chat.create_chat_completion(req)
1512
1513	assert mock_engine.generate.call_args.args[1].max_tokens == 10
1514
1515	# Test Case 3.2: Request max_tokens higher — capped by user ceiling from override
1516	req.max_tokens = 15
1517
1518	with suppress(Exception):
1519	await serving_chat.create_chat_completion(req)
1520
1521	assert mock_engine.generate.call_args.args[1].max_tokens == 10
1522
1523	# Test Case 3.3: Request max_tokens lower — respected
1524	req.max_tokens = 5
1525
1526	with suppress(Exception):
1527	await serving_chat.create_chat_completion(req)
1528
1529	assert mock_engine.generate.call_args.args[1].max_tokens == 5
1530
1531	# Setting server's max_tokens in the generation_config.json
1532	# higher than context_window - prompt_tokens
1533	mock_model_config = MockModelConfig()
1534	mock_model_config.diff_sampling_param = {"max_tokens": 200}
1535
1536	# Reinitialize the engine with new settings
1537	mock_engine = MagicMock(spec=AsyncLLM)
1538	mock_engine.errored = False
1539	mock_engine.model_config = mock_model_config
1540	mock_engine.input_processor = MagicMock()
1541	mock_engine.renderer = _build_renderer(mock_engine.model_config)
1542
1543	# Initialize the serving chat
1544	serving_chat = _build_serving_chat(mock_engine)
1545
1546	# Test case 1: No max_tokens specified, defaults to context_window
1547	req = ChatCompletionRequest(
1548	model=MODEL_NAME,
1549	messages=[{"role": "user", "content": "what is 1+1?"}],
1550	)
1551
1552	with suppress(Exception):
1553	await serving_chat.create_chat_completion(req)
1554
1555	assert mock_engine.generate.call_args.args[1].max_tokens == 93
1556
1557	# Test Case 2: Request's max_tokens set higher than server accepts
1558	req.max_tokens = 100
1559
1560	with suppress(Exception):
1561	await serving_chat.create_chat_completion(req)
1562
1563	assert mock_engine.generate.call_args.args[1].max_tokens == 93
1564
1565	# Test Case 3: Request's max_tokens set lower than server accepts
1566	req.max_tokens = 5
1567
1568	with suppress(Exception):
1569	await serving_chat.create_chat_completion(req)
1570
1571	assert mock_engine.generate.call_args.args[1].max_tokens == 5
1572
1573
1574	@pytest.mark.asyncio
1575	async def test_serving_chat_truncate_prompt_tokens_max_token_accounting():
1576	"""When truncate_prompt_tokens is set, max_tokens must be calculated using
1577	the truncated prompt length, not the original prompt length.
1578
1579	Regression: without the fix, get_max_tokens received the untruncated prompt
1580	length, causing the output budget to be underestimated.
1581	"""
1582	mock_engine = MagicMock(spec=AsyncLLM)
1583	mock_engine.errored = False
1584	mock_engine.model_config = MockModelConfig()
1585	mock_engine.input_processor = MagicMock()
1586	mock_engine.renderer = _build_renderer(mock_engine.model_config)
1587
1588	serving_chat = _build_serving_chat(mock_engine)
1589
1590	# "what is 1+1?" tokenizes to 7 tokens with the test chat template
1591	# (max_model_len=100 -> max_tokens = 93 without truncation, confirmed by
1592	# test_serving_chat_should_set_correct_max_tokens above).
1593	messages = [{"role": "user", "content": "what is 1+1?"}]
1594
1595	# Baseline: no truncation -> max_tokens = 100 - 7 = 93.
1596	req = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
1597	with suppress(Exception):
1598	await serving_chat.create_chat_completion(req)
1599	assert mock_engine.generate.call_args.args[1].max_tokens == 93
1600
1601	# With truncate_prompt_tokens=5 (less than 7): the effective prompt length
1602	# is 5, so max_tokens should be 100 - 5 = 95, not 93.
1603	req = ChatCompletionRequest(
1604	model=MODEL_NAME,
1605	messages=messages,
1606	truncate_prompt_tokens=5,
1607	)
1608	with suppress(Exception):
1609	await serving_chat.create_chat_completion(req)
1610	assert mock_engine.generate.call_args.args[1].max_tokens == 95
1611
1612	# With truncate_prompt_tokens=-1 (meaning use full max_model_len as the
1613	# truncation limit, i.e., no practical truncation vs the window): effective
1614	# length = min(7, 100) = 7 -> max_tokens = 93 again.
1615	req = ChatCompletionRequest(
1616	model=MODEL_NAME,
1617	messages=messages,
1618	truncate_prompt_tokens=-1,
1619	)
1620	with suppress(Exception):
1621	await serving_chat.create_chat_completion(req)
1622	assert mock_engine.generate.call_args.args[1].max_tokens == 93
1623
1624
1625	@pytest.mark.asyncio
1626	async def test_serving_chat_truncation_side_controls_prompt_truncation():
1627	model_config = MockModelConfig()
1628	model_config.model = MODEL_NAME_SHORT
1629	model_config.tokenizer = MODEL_NAME_SHORT
1630	mock_engine = MockEngine(
1631	model_config=model_config,
1632	renderer=_build_renderer(model_config),
1633	)
1634
1635	serving_chat = _build_serving_chat(mock_engine)
1636	messages = [
1637	{
1638	"role": "user",
1639	"content": "Summarize how prompt truncation works in one sentence.",
1640	}
1641	]
1642
1643	full_token_ids = await _render_chat_prompt_token_ids(
1644	serving_chat,
1645	messages,
1646	model_name=MODEL_NAME_SHORT,
1647	)
1648	assert len(full_token_ids) > 4
1649
1650	right_token_ids = await _render_chat_prompt_token_ids(
1651	serving_chat,
1652	messages,
1653	model_name=MODEL_NAME_SHORT,
1654	truncate_prompt_tokens=4,
1655	truncation_side="right",
1656	)
1657	assert right_token_ids == full_token_ids[:4]
1658
1659	left_token_ids = await _render_chat_prompt_token_ids(
1660	serving_chat,
1661	messages,
1662	model_name=MODEL_NAME_SHORT,
1663	truncate_prompt_tokens=4,
1664	truncation_side="left",
1665	)
1666	assert left_token_ids == full_token_ids[-4:]
1667
1668
1669	def test_serving_chat_starts_without_tokenizer():
1670	"""Regression test: building the serving objects must not require a
1671	tokenizer, which is absent under `skip_tokenizer_init`."""
1672	mock_engine = _build_mock_engine()
1673	mock_engine.model_config = MockModelConfig(skip_tokenizer_init=True)
1674	mock_engine.renderer = HfRenderer(
1675	MockVllmConfig(mock_engine.model_config, parallel_config=MockParallelConfig()),
1676	None,
1677	)
1678
1679	serving_chat = _build_serving_chat(mock_engine)
1680
1681	assert serving_chat.parser_cls is None
1682
1683
1684	@pytest.mark.asyncio
1685	async def test_serving_chat_mistral_token_ids_prompt_is_validated():
1686	"""Regression test: when the Mistral tokenizer path returns token IDs
1687	directly, we must still apply input length + max_tokens validation.
1688	"""
1689	mock_engine = MagicMock(spec=AsyncLLM)
1690	mock_engine.errored = False
1691	mock_engine.model_config = MockModelConfig(skip_tokenizer_init=True)
1692	mock_engine.input_processor = MagicMock()
1693
1694	mock_tokenizer = MagicMock(spec=MistralTokenizer)
1695	mock_renderer = MistralRenderer(
1696	MockVllmConfig(mock_engine.model_config, parallel_config=MockParallelConfig()),
1697	tokenizer=mock_tokenizer,
1698	)
1699	# Force the Mistral chat template renderer to return token IDs.
1700	# Choose a prompt length that is < max_model_len, but large enough that
1701	# adding max_tokens should exceed the model context window.
1702	mock_renderer.render_messages_async = AsyncMock(
1703	return_value=(
1704	[],
1705	TokensPrompt(prompt_token_ids=list(range(95))),
1706	)
1707	)
1708	mock_engine.renderer = mock_renderer
1709
1710	serving_chat = _build_serving_chat(mock_engine)
1711
1712	req = ChatCompletionRequest(
1713	model=MODEL_NAME,
1714	messages=[{"role": "user", "content": "what is 1+1?"}],
1715	max_tokens=10,
1716	)
1717
1718	with pytest.raises(VLLMValidationError):
1719	await serving_chat.create_chat_completion(req)
1720
1721
1722	@pytest.mark.asyncio
1723	async def test_serving_chat_mistral_token_ids_prompt_too_long_is_rejected():
1724	"""Regression test: MistralTokenizer token-id prompts must still enforce
1725	the max context length for the input itself (token_num >= max_model_len).
1726	"""
1727	mock_engine = MagicMock(spec=AsyncLLM)
1728	mock_engine.errored = False
1729	mock_engine.model_config = MockModelConfig(skip_tokenizer_init=True)
1730	mock_engine.input_processor = MagicMock()
1731
1732	mock_tokenizer = MagicMock(spec=MistralTokenizer)
1733	mock_renderer = MistralRenderer(
1734	MockVllmConfig(mock_engine.model_config, parallel_config=MockParallelConfig()),
1735	tokenizer=mock_tokenizer,
1736	)
1737	# prompt_token_ids length == max_model_len should be rejected for
1738	# completion-like requests (ChatCompletionRequest).
1739	mock_renderer.render_messages_async = AsyncMock(
1740	return_value=(
1741	[],
1742	TokensPrompt(
1743	prompt_token_ids=list(range(mock_engine.model_config.max_model_len))
1744	),
1745	)
1746	)
1747	mock_engine.renderer = mock_renderer
1748
1749	serving_chat = _build_serving_chat(mock_engine)
1750
1751	req = ChatCompletionRequest(
1752	model=MODEL_NAME,
1753	messages=[{"role": "user", "content": "what is 1+1?"}],
1754	max_tokens=1,
1755	)
1756
1757	with pytest.raises(VLLMValidationError):
1758	await serving_chat.create_chat_completion(req)
1759
1760
1761	@pytest.mark.asyncio
1762	async def test_serving_chat_could_load_correct_generation_config():
1763	mock_model_config = MockModelConfig()
1764	mock_model_config.diff_sampling_param = {
1765	"temperature": 0.5,
1766	"repetition_penalty": 1.05,
1767	}
1768
1769	mock_engine = MagicMock(spec=AsyncLLM)
1770	mock_engine.errored = False
1771	mock_engine.model_config = mock_model_config
1772	mock_engine.input_processor = MagicMock()
1773	mock_engine.renderer = _build_renderer(mock_engine.model_config)
1774
1775	# Initialize the serving chat
1776	serving_chat = _build_serving_chat(mock_engine)
1777
1778	req = ChatCompletionRequest(
1779	model=MODEL_NAME,
1780	messages=[{"role": "user", "content": "what is 1+1?"}],
1781	)
1782
1783	with suppress(Exception):
1784	await serving_chat.create_chat_completion(req)
1785
1786	assert mock_engine.generate.call_args.args[1].temperature == 0.5
1787	assert mock_engine.generate.call_args.args[1].repetition_penalty == 1.05
1788
1789	# Test the param when user set it
1790	req.temperature = 0.1
1791
1792	with suppress(Exception):
1793	await serving_chat.create_chat_completion(req)
1794
1795	assert mock_engine.generate.call_args.args[1].temperature == 0.1
1796	assert mock_engine.generate.call_args.args[1].repetition_penalty == 1.05
1797
1798	# Test When temperature==0.0
1799	req.temperature = 0.0
1800
1801	with suppress(Exception):
1802	await serving_chat.create_chat_completion(req)
1803
1804	assert mock_engine.generate.call_args.args[1].temperature == 0.0
1805	assert mock_engine.generate.call_args.args[1].repetition_penalty == 1.05
1806
1807
1808	@pytest.mark.parametrize("model_type", ["gpt_oss", "any"])
1809	@pytest.mark.asyncio
1810	async def test_serving_chat_did_set_correct_cache_salt(model_type):
1811	mock_model_config = MockModelConfig()
1812	mock_model_config.hf_config.model_type = model_type
1813
1814	mock_engine = MagicMock(spec=AsyncLLM)
1815	mock_engine.errored = False
1816	mock_engine.model_config = mock_model_config
1817	mock_engine.input_processor = MagicMock()
1818	mock_engine.renderer = _build_renderer(mock_engine.model_config)
1819
1820	serving_chat = _build_serving_chat(mock_engine)
1821
1822	orig_render_chat_request = serving_chat.render_chat_request
1823	captured_inputs = []
1824
1825	async def render_chat_request(request):
1826	result = await orig_render_chat_request(request)
1827
1828	assert isinstance(result, tuple)
1829	conversation, engine_inputs = result
1830	captured_inputs.extend(engine_inputs)
1831
1832	return result
1833
1834	serving_chat.render_chat_request = render_chat_request
1835
1836	# Test cache_salt
1837	req = ChatCompletionRequest(
1838	model=MODEL_NAME,
1839	messages=[{"role": "user", "content": "what is 1+1?"}],
1840	)
1841
1842	# By default, cache_salt in the engine prompt is not set
1843	with suppress(Exception):
1844	await serving_chat.create_chat_completion(req)
1845
1846	assert len(captured_inputs) == 1
1847	assert "cache_salt" not in captured_inputs[0]
1848
1849	captured_inputs.clear()
1850
1851	# Test with certain cache_salt
1852	req.cache_salt = "test_salt"
1853	with suppress(Exception):
1854	await serving_chat.create_chat_completion(req)
1855
1856	assert len(captured_inputs) == 1
1857	assert captured_inputs[0]["cache_salt"] == "test_salt"
1858
1859
1860	@pytest.mark.asyncio
1861	async def test_serving_chat_data_parallel_rank_extraction():
1862	"""Test that data_parallel_rank is properly extracted from header and
1863	passed to engine."""
1864	mock_engine = MagicMock(spec=AsyncLLM)
1865	mock_engine.errored = False
1866	mock_engine.model_config = MockModelConfig()
1867	mock_engine.input_processor = MagicMock()
1868	mock_engine.renderer = _build_renderer(mock_engine.model_config)
1869
1870	# Mock the generate method to return an async generator
1871	async def mock_generate(args, *kwargs):
1872	# Yield a fake RequestOutput
1873	from vllm.outputs import CompletionOutput, RequestOutput
1874
1875	yield RequestOutput(
1876	request_id="test-request",
1877	prompt="test prompt",
1878	prompt_token_ids=[1, 2, 3],
1879	prompt_logprobs=None,
1880	outputs=[
1881	CompletionOutput(
1882	index=0,
1883	text="test response",
1884	token_ids=[4, 5, 6],
1885	cumulative_logprob=0.0,
1886	logprobs=None,
1887	finish_reason="stop",
1888	stop_reason=None,
1889	)
1890	],
1891	finished=True,
1892	)
1893
1894	mock_engine.generate = AsyncMock(side_effect=mock_generate)
1895
1896	serving_chat = _build_serving_chat(mock_engine)
1897
1898	# Test when data_parallel_rank is present in header
1899	req = ChatCompletionRequest(
1900	model=MODEL_NAME,
1901	messages=[{"role": "user", "content": "what is 1+1?"}],
1902	)
1903
1904	# Mock request with X-data-parallel-rank header
1905	mock_raw_request = MagicMock()
1906	mock_raw_request.headers = {"X-data-parallel-rank": "2"}
1907	mock_raw_request.state = MagicMock()
1908
1909	with suppress(Exception):
1910	await serving_chat.create_chat_completion(req, mock_raw_request)
1911
1912	# Verify that data_parallel_rank was passed to engine.generate
1913	assert "data_parallel_rank" in mock_engine.generate.call_args.kwargs
1914	assert mock_engine.generate.call_args.kwargs["data_parallel_rank"] == 2
1915
1916	# Test when data_parallel_rank is not present (defaults to None)
1917	req_no_dp = ChatCompletionRequest(
1918	model=MODEL_NAME,
1919	messages=[{"role": "user", "content": "what is 2+2?"}],
1920	)
1921
1922	# Mock request with no header
1923	mock_raw_request_no_dp = MagicMock()
1924	mock_raw_request_no_dp.headers = {}
1925	mock_raw_request_no_dp.state = MagicMock()
1926
1927	with suppress(Exception):
1928	await serving_chat.create_chat_completion(req_no_dp, mock_raw_request_no_dp)
1929
1930	# Verify that data_parallel_rank defaults to None
1931	assert "data_parallel_rank" in mock_engine.generate.call_args.kwargs
1932	assert mock_engine.generate.call_args.kwargs["data_parallel_rank"] is None
1933
1934
1935	async def _render_chat_prompt_token_ids(
1936	serving_chat: OpenAIServingChat,
1937	messages: list[dict[str, str]],
1938	*,
1939	model_name: str = MODEL_NAME,
1940	truncate_prompt_tokens: int \| None = None,
1941	truncation_side: str \| None = None,
1942	) -> list[int]:
1943	request = ChatCompletionRequest(
1944	model=model_name,
1945	messages=messages,
1946	max_tokens=1,
1947	temperature=0.0,
1948	return_token_ids=True,
1949	truncate_prompt_tokens=truncate_prompt_tokens,
1950	truncation_side=truncation_side,
1951	)
1952
1953	result = await serving_chat.render_chat_request(request)
1954	assert not isinstance(result, ErrorResponse)
1955
1956	_, engine_inputs = result
1957	assert len(engine_inputs) == 1
1958
1959	prompt_token_ids = serving_chat._extract_prompt_components(
1960	engine_inputs[0]
1961	).token_ids
1962	assert prompt_token_ids is not None
1963	return prompt_token_ids
1964
1965
1966	class TestServingChatWithHarmony:
1967	"""These tests ensure Chat Completion requests are being properly converted into
1968	Harmony messages and Harmony response messages back into Chat Completion responses.
1969	These tests are not exhaustive, but each one was created to cover a specific case
1970	that we got wrong but is now fixed.
1971
1972	Any changes to the tests and their expectations may result in changes to the
1973	accuracy of model prompting and responses generated. It is suggested to run
1974	an evaluation or benchmarking suite (such as bfcl multi_turn) to understand
1975	any impact of changes in how we prompt Harmony models.
1976	"""
1977
1978	@pytest.fixture(params=[False, True], ids=["non_streaming", "streaming"])
1979	def stream(self, request) -> bool:
1980	"""Parameterize tests to run in both non-streaming and streaming modes."""
1981	return request.param
1982
1983	@pytest.fixture()
1984	def mock_engine(self) -> AsyncLLM:
1985	mock_engine = MagicMock(spec=AsyncLLM)
1986	mock_engine.errored = False
1987	mock_engine.model_config = MockModelConfig()
1988	mock_engine.model_config.hf_config = MockHFConfig(model_type="gpt_oss")
1989	mock_engine.model_config.hf_text_config = MockHFConfig(model_type="gpt_oss")
1990	mock_engine.input_processor = MagicMock()
1991	mock_engine.renderer = _build_renderer(mock_engine.model_config)
1992	return mock_engine
1993
1994	@pytest.fixture()
1995	def serving_chat(self, mock_engine) -> OpenAIServingChat:
1996	chat = _build_serving_chat(
1997	mock_engine,
1998	reasoning_parser="openai_gptoss",
1999	tool_parser="openai",
2000	enable_auto_tools=True,
2001	)
2002	assert chat.parser_cls is HarmonyParser
2003	return chat
2004
2005	def mock_request_output_from_req_and_token_ids(
2006	self, req: ChatCompletionRequest, token_ids: list[int], finished: bool = False
2007	) -> RequestOutput:
2008	# Our tests don't use most fields, so just get the token ids correct
2009	completion_output = CompletionOutput(
2010	index=0,
2011	text="",
2012	token_ids=token_ids,
2013	cumulative_logprob=0.0,
2014	logprobs=None,
2015	)
2016	return RequestOutput(
2017	request_id=req.request_id,
2018	prompt=[],
2019	prompt_token_ids=[],
2020	prompt_logprobs=None,
2021	outputs=[completion_output],
2022	finished=finished,
2023	)
2024
2025	@pytest.fixture
2026	def weather_tools(self) -> list[dict[str, Any]]:
2027	return [
2028	{
2029	"type": "function",
2030	"function": {
2031	"name": "get_weather",
2032	"description": "Get the weather in a given location",
2033	"parameters": {
2034	"type": "object",
2035	"properties": {
2036	"location": {"type": "string"},
2037	},
2038	"required": ["location"],
2039	},
2040	},
2041	},
2042	]
2043
2044	@pytest.fixture
2045	def weather_messages_start(self) -> list[dict[str, Any]]:
2046	return [
2047	{
2048	"role": "user",
2049	"content": "What's the weather like in Paris today?",
2050	},
2051	]
2052
2053	async def generate_response_from_harmony_str(
2054	self,
2055	serving_chat: OpenAIServingChat,
2056	req: ChatCompletionRequest,
2057	harmony_str: str,
2058	stream: bool = False,
2059	) -> ChatCompletionResponse:
2060	harmony_token_ids = get_encoding().encode(harmony_str, allowed_special="all")
2061	tokenizer = get_tokenizer(GPT_OSS_MODEL_NAME)
2062
2063	async def result_generator():
2064	if stream:
2065	for token_id in harmony_token_ids:
2066	yield self.mock_request_output_from_req_and_token_ids(
2067	req, [token_id]
2068	)
2069	yield self.mock_request_output_from_req_and_token_ids(
2070	req, [], finished=True
2071	)
2072	else:
2073	yield self.mock_request_output_from_req_and_token_ids(
2074	req, harmony_token_ids, finished=True
2075	)
2076
2077	generator_func = (
2078	serving_chat.chat_completion_stream_generator
2079	if stream
2080	else serving_chat.chat_completion_full_generator
2081	)
2082
2083	chat_template_kwargs = serving_chat._effective_chat_template_kwargs(req)
2084	if stream:
2085	extra_kwargs: dict[str, Any] = {
2086	"chat_template_kwargs": chat_template_kwargs,
2087	}
2088	else:
2089	parser = None
2090	if serving_chat.parser_cls is not None:
2091	parser = serving_chat.parser_cls(
2092	tokenizer,
2093	req.tools,
2094	chat_template_kwargs=chat_template_kwargs,
2095	)
2096	extra_kwargs = {"parser": parser}
2097
2098	result = generator_func(
2099	request=req,
2100	result_generator=result_generator(),
2101	request_id=req.request_id,
2102	model_name=req.model,
2103	conversation=[],
2104	tokenizer=tokenizer,
2105	request_metadata=RequestResponseMetadata(
2106	request_id=req.request_id,
2107	model_name=req.model,
2108	),
2109	**extra_kwargs,
2110	)
2111
2112	if stream:
2113	return await accumulate_streaming_response(result)
2114	return await result
2115
2116	@pytest.mark.asyncio
2117	@pytest.mark.parametrize(
2118	"include_reasoning", [True, False], ids=["with_reasoning", "no_reasoning"]
2119	)
2120	async def test_simple_chat(self, serving_chat, stream, include_reasoning):
2121	messages = [{"role": "user", "content": "what is 1+1?"}]
2122
2123	# Test the Harmony messages for the first turn's input
2124	req = ChatCompletionRequest(
2125	model=MODEL_NAME,
2126	messages=messages,
2127	include_reasoning=include_reasoning,
2128	)
2129	input_messages, _ = serving_chat.online_renderer._make_request_with_harmony(req)
2130	verify_harmony_messages(
2131	input_messages,
2132	[
2133	{"role": "system"},
2134	{"role": "user", "content": messages[0]["content"]},
2135	],
2136	)
2137
2138	# Test the Chat Completion response for the first turn's output
2139	reasoning_str = "We need to think really hard about this."
2140	final_str = "The answer is 2."
2141	response_str = (
2142	f"<\|channel\|>analysis<\|message\|>{reasoning_str}<\|end\|>"
2143	f"<\|start\|>assistant<\|channel\|>final<\|message\|>{final_str}<\|end\|>"
2144	)
2145	response = await self.generate_response_from_harmony_str(
2146	serving_chat, req, response_str, stream=stream
2147	)
2148	verify_chat_response(
2149	response,
2150	content=final_str,
2151	reasoning=reasoning_str if include_reasoning else None,
2152	)
2153
2154	# Add the output messages from the first turn as input to the second turn
2155	for choice in response.choices:
2156	messages.append(choice.message.model_dump(exclude_none=True))
2157
2158	# Test the Harmony messages for the second turn's input
2159	req_2 = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
2160	input_messages_2, _ = serving_chat.online_renderer._make_request_with_harmony(
2161	req_2
2162	)
2163	expected_input_messages_2 = [
2164	{"role": "system"},
2165	{"role": "user"},
2166	]
2167	if include_reasoning:
2168	expected_input_messages_2.append(
2169	{
2170	"role": "assistant",
2171	"channel": "analysis",
2172	}
2173	)
2174	expected_input_messages_2.append(
2175	{"role": "assistant", "channel": "final", "content": final_str}
2176	)
2177	verify_harmony_messages(
2178	input_messages_2,
2179	expected_input_messages_2,
2180	)
2181
2182	@pytest.mark.asyncio
2183	async def test_system_message_without_tools(self, serving_chat, stream):
2184	"""Leading system message produces a developer message with
2185	DeveloperContent (# Instructions header)."""
2186	messages = [
2187	{"role": "system", "content": "You are a helpful assistant."},
2188	{"role": "user", "content": "Hello"},
2189	]
2190	req = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
2191	input_messages, _ = serving_chat.online_renderer._make_request_with_harmony(req)
2192	verify_harmony_messages(
2193	input_messages,
2194	[
2195	{"role": "system"},
2196	{
2197	"role": "developer",
2198	"instructions": "You are a helpful assistant.",
2199	},
2200	{"role": "user", "content": "Hello"},
2201	],
2202	)
2203
2204	@pytest.mark.asyncio
2205	async def test_system_message_with_tools(self, serving_chat, stream, weather_tools):
2206	"""Leading system message is folded into the developer message
2207	alongside tool definitions."""
2208	messages = [
2209	{"role": "system", "content": "You are a helpful assistant."},
2210	{"role": "user", "content": "What's the weather?"},
2211	]
2212	req = ChatCompletionRequest(
2213	model=MODEL_NAME, messages=messages, tools=weather_tools
2214	)
2215	input_messages, _ = serving_chat.online_renderer._make_request_with_harmony(req)
2216	verify_harmony_messages(
2217	input_messages,
2218	[
2219	{"role": "system"},
2220	{
2221	"role": "developer",
2222	"instructions": "You are a helpful assistant.",
2223	"tool_definitions": ["get_weather"],
2224	},
2225	{"role": "user", "content": "What's the weather?"},
2226	],
2227	)
2228
2229	@pytest.mark.asyncio
2230	async def test_tool_call_response_with_content(
2231	self, serving_chat, stream, weather_tools, weather_messages_start
2232	):
2233	tools = weather_tools
2234	messages = list(weather_messages_start)
2235
2236	# Test the Harmony messages for the first turn's input
2237	req = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools)
2238	input_messages, _ = serving_chat.online_renderer._make_request_with_harmony(req)
2239	verify_harmony_messages(
2240	input_messages,
2241	[
2242	{"role": "system"},
2243	{"role": "developer", "tool_definitions": ["get_weather"]},
2244	{"role": "user", "content": messages[0]["content"]},
2245	],
2246	)
2247
2248	# Test the Chat Completion response for the first turn's output
2249	commentary_str = "We'll call get_weather."
2250	tool_args_str = '{"location": "Paris"}'
2251	response_str = (
2252	f"<\|channel\|>commentary<\|message\|>{commentary_str}<\|end\|>"
2253	"<\|start\|>assistant to=functions.get_weather<\|channel\|>commentary"
2254	f"<\|constrain\|>json<\|message\|>{tool_args_str}<\|call\|>"
2255	)
2256	response = await self.generate_response_from_harmony_str(
2257	serving_chat, req, response_str, stream=stream
2258	)
2259	verify_chat_response(
2260	response,
2261	content=commentary_str,
2262	tool_calls=[("get_weather", tool_args_str)],
2263	)
2264
2265	tool_call = response.choices[0].message.tool_calls[0]
2266
2267	# Add the output messages from the first turn as input to the second turn
2268	for choice in response.choices:
2269	messages.append(choice.message.model_dump(exclude_none=True))
2270
2271	# Add our tool output message
2272	messages.append(
2273	{
2274	"role": "tool",
2275	"tool_call_id": tool_call.id,
2276	"content": "20 degrees Celsius",
2277	},
2278	)
2279
2280	# Test the Harmony messages for the second turn's input
2281	req_2 = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools)
2282	input_messages_2, _ = serving_chat.online_renderer._make_request_with_harmony(
2283	req_2
2284	)
2285	verify_harmony_messages(
2286	input_messages_2,
2287	[
2288	{"role": "system"},
2289	{"role": "developer"},
2290	{"role": "user"},
2291	{
2292	"role": "assistant",
2293	"channel": "commentary",
2294	"content": commentary_str,
2295	},
2296	{
2297	"role": "assistant",
2298	"channel": "commentary",
2299	"recipient": "functions.get_weather",
2300	"content": tool_args_str,
2301	},
2302	{
2303	"role": "tool",
2304	"author_name": "functions.get_weather",
2305	"channel": "commentary",
2306	"recipient": "assistant",
2307	"content": "20 degrees Celsius",
2308	},
2309	],
2310	)
2311
2312	@pytest.mark.asyncio
2313	async def test_multi_turn_tools_and_reasoning(
2314	self, serving_chat, stream, weather_tools, weather_messages_start
2315	):
2316	tools = weather_tools
2317	messages = list(weather_messages_start)
2318
2319	# Test the Harmony messages for the first turn's input
2320	req = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools)
2321	input_messages, _ = serving_chat.online_renderer._make_request_with_harmony(req)
2322	verify_harmony_messages(
2323	input_messages,
2324	[
2325	{"role": "system"},
2326	{"role": "developer", "tool_definitions": ["get_weather"]},
2327	{"role": "user", "content": messages[0]["content"]},
2328	],
2329	)
2330
2331	# Test the Chat Completion response for the first turn's output
2332	reasoning_str = "I'll call get_weather."
2333	paris_tool_args_str = '{"location": "Paris"}'
2334	response_str = (
2335	f"<\|channel\|>analysis<\|message\|>{reasoning_str}<\|end\|>"
2336	"<\|start\|>assistant to=functions.get_weather<\|channel\|>commentary"
2337	f"<\|constrain\|>json<\|message\|>{paris_tool_args_str}<\|call\|>"
2338	)
2339	response = await self.generate_response_from_harmony_str(
2340	serving_chat, req, response_str, stream=stream
2341	)
2342	verify_chat_response(
2343	response,
2344	reasoning=reasoning_str,
2345	tool_calls=[("get_weather", paris_tool_args_str)],
2346	)
2347
2348	tool_call = response.choices[0].message.tool_calls[0]
2349
2350	# Add the output messages from the first turn as input to the second turn
2351	for choice in response.choices:
2352	messages.append(choice.message.model_dump(exclude_none=True))
2353
2354	# Add our tool output message
2355	messages.append(
2356	{
2357	"role": "tool",
2358	"tool_call_id": tool_call.id,
2359	"content": "20 degrees Celsius",
2360	},
2361	)
2362
2363	# Test the Harmony messages for the second turn's input
2364	req_2 = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools)
2365	input_messages_2, _ = serving_chat.online_renderer._make_request_with_harmony(
2366	req_2
2367	)
2368	verify_harmony_messages(
2369	input_messages_2,
2370	[
2371	{"role": "system"},
2372	{"role": "developer"},
2373	{"role": "user"},
2374	{
2375	"role": "assistant",
2376	"channel": "analysis",
2377	},
2378	{
2379	"role": "assistant",
2380	"channel": "commentary",
2381	"recipient": "functions.get_weather",
2382	"content": paris_tool_args_str,
2383	},
2384	{
2385	"role": "tool",
2386	"author_name": "functions.get_weather",
2387	"channel": "commentary",
2388	"recipient": "assistant",
2389	"content": "20 degrees Celsius",
2390	},
2391	],
2392	)
2393
2394	# Test the Chat Completion response for the second turn's output
2395	paris_weather_str = "The weather in Paris today is 20 degrees Celsius."
2396	response_str = f"<\|channel\|>final<\|message\|>{paris_weather_str}<\|end\|>"
2397	response_2 = await self.generate_response_from_harmony_str(
2398	serving_chat, req_2, response_str, stream=stream
2399	)
2400	verify_chat_response(response_2, content=paris_weather_str)
2401
2402	# Add the output messages from the second turn as input to the third turn
2403	for choice in response_2.choices:
2404	messages.append(choice.message.model_dump(exclude_none=True))
2405
2406	# Add a new user message for the third turn
2407	messages.append(
2408	{
2409	"role": "user",
2410	"content": "What's the weather like in Boston today?",
2411	},
2412	)
2413
2414	# Test the Harmony messages for the third turn's input
2415	req_3 = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools)
2416	input_messages_3, _ = serving_chat.online_renderer._make_request_with_harmony(
2417	req_3
2418	)
2419	verify_harmony_messages(
2420	input_messages_3,
2421	[
2422	{"role": "system"},
2423	{"role": "developer"},
2424	{"role": "user"},
2425	{
2426	"role": "assistant",
2427	"channel": "analysis",
2428	"content": reasoning_str,
2429	},
2430	{
2431	"role": "assistant",
2432	"channel": "commentary",
2433	"recipient": "functions.get_weather",
2434	"content": paris_tool_args_str,
2435	},
2436	{
2437	"role": "tool",
2438	"author_name": "functions.get_weather",
2439	"channel": "commentary",
2440	"recipient": "assistant",
2441	"content": "20 degrees Celsius",
2442	},
2443	{
2444	"role": "assistant",
2445	"channel": "final",
2446	"content": paris_weather_str,
2447	},
2448	{"role": "user", "content": messages[-1]["content"]},
2449	],
2450	)
2451
2452	# Test the Chat Completion response for the third turn's output
2453	reasoning_str = "I'll call get_weather."
2454	boston_tool_args_str = '{"location": "Boston"}'
2455	response_str = (
2456	f"<\|channel\|>analysis<\|message\|>{reasoning_str}<\|end\|>"
2457	"<\|start\|>assistant to=functions.get_weather<\|channel\|>commentary"
2458	f"<\|constrain\|>json<\|message\|>{boston_tool_args_str}<\|call\|>"
2459	)
2460	response_3 = await self.generate_response_from_harmony_str(
2461	serving_chat, req, response_str, stream=stream
2462	)
2463	verify_chat_response(
2464	response_3,
2465	reasoning=reasoning_str,
2466	tool_calls=[("get_weather", boston_tool_args_str)],
2467	)
2468
2469	tool_call = response_3.choices[0].message.tool_calls[0]
2470
2471	# Add the output messages from the third turn as input to the fourth turn
2472	for choice in response_3.choices:
2473	messages.append(choice.message.model_dump(exclude_none=True))
2474
2475	# Add our tool output message
2476	messages.append(
2477	{
2478	"role": "tool",
2479	"tool_call_id": tool_call.id,
2480	"content": "10 degrees Celsius",
2481	},
2482	)
2483
2484	# Test the Harmony messages for the fourth turn's input
2485	req_4 = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools)
2486	input_messages_4, _ = serving_chat.online_renderer._make_request_with_harmony(
2487	req_4
2488	)
2489	verify_harmony_messages(
2490	input_messages_4,
2491	[
2492	{"role": "system"},
2493	{"role": "developer"},
2494	{"role": "user"},
2495	{
2496	"role": "assistant",
2497	"channel": "analysis",
2498	},
2499	{"role": "assistant"},
2500	{"role": "tool"},
2501	{
2502	"role": "assistant",
2503	"channel": "final",
2504	},
2505	{"role": "user"},
2506	{
2507	"role": "assistant",
2508	"channel": "analysis",
2509	"content": reasoning_str,
2510	},
2511	{
2512	"role": "assistant",
2513	"channel": "commentary",
2514	"recipient": "functions.get_weather",
2515	"content": boston_tool_args_str,
2516	},
2517	{
2518	"role": "tool",
2519	"author_name": "functions.get_weather",
2520	"channel": "commentary",
2521	"recipient": "assistant",
2522	"content": "10 degrees Celsius",
2523	},
2524	],
2525	)
2526
2527	@pytest.mark.asyncio
2528	async def test_non_tool_reasoning(self, serving_chat):
2529	messages: list[dict[str, Any]] = [
2530	{
2531	"role": "user",
2532	"content": "What's 2+2?",
2533	},
2534	{
2535	"role": "assistant",
2536	"reasoning": "Adding 2 and 2 is easy. The result is 4.",
2537	"content": "4",
2538	},
2539	]
2540	req = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
2541	input_messages, _ = serving_chat.online_renderer._make_request_with_harmony(req)
2542
2543	verify_harmony_messages(
2544	input_messages,
2545	[
2546	{"role": "system"},
2547	{"role": "user", "content": messages[0]["content"]},
2548	{
2549	"role": "assistant",
2550	"channel": "analysis",
2551	},
2552	{
2553	"role": "assistant",
2554	"channel": "final",
2555	"content": messages[1]["content"],
2556	},
2557	],
2558	)
2559
2560	@pytest.mark.asyncio
2561	async def test_non_tool_reasoning_empty_content(self, serving_chat):
2562	messages: list[dict[str, Any]] = [
2563	{
2564	"role": "user",
2565	"content": "What's 2+2?",
2566	},
2567	{
2568	"role": "assistant",
2569	"reasoning": "Adding 2 and 2 is easy. The result is 4.",
2570	"content": "",
2571	},
2572	]
2573	req = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
2574	input_messages, _ = serving_chat.online_renderer._make_request_with_harmony(req)
2575
2576	verify_harmony_messages(
2577	input_messages,
2578	[
2579	{"role": "system"},
2580	{"role": "user", "content": messages[0]["content"]},
2581	{
2582	"role": "assistant",
2583	"channel": "analysis",
2584	"content": messages[1]["reasoning"],
2585	},
2586	],
2587	)
2588
2589	@pytest.mark.asyncio
2590	async def test_non_tool_reasoning_empty_content_list(self, serving_chat):
2591	messages: list[dict[str, Any]] = [
2592	{
2593	"role": "user",
2594	"content": "What's 2+2?",
2595	},
2596	{
2597	"role": "assistant",
2598	"reasoning": "Adding 2 and 2 is easy. The result is 4.",
2599	"content": [],
2600	},
2601	]
2602	req = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
2603	input_messages, _ = serving_chat.online_renderer._make_request_with_harmony(req)
2604
2605	verify_harmony_messages(
2606	input_messages,
2607	[
2608	{"role": "system"},
2609	{"role": "user", "content": messages[0]["content"]},
2610	{
2611	"role": "assistant",
2612	"channel": "analysis",
2613	"content": messages[1]["reasoning"],
2614	},
2615	],
2616	)
2617
2618
2619	@pytest.mark.asyncio
2620	async def test_tool_choice_validation_without_parser():
2621	"""Test that tool_choice='required' or named tool without tool_parser
2622	returns an appropriate error message."""
2623	mock_engine = MagicMock(spec=AsyncLLM)
2624	mock_engine.errored = False
2625	mock_engine.model_config = MockModelConfig()
2626	mock_engine.input_processor = MagicMock()
2627	mock_engine.renderer = _build_renderer(mock_engine.model_config)
2628
2629	models = OpenAIServingModels(
2630	engine_client=mock_engine,
2631	base_model_paths=BASE_MODEL_PATHS,
2632	)
2633	online_renderer = _build_online_renderer(mock_engine, models.registry)
2634
2635	# Create serving_chat without tool_parser (enable_auto_tools=False)
2636	serving_chat = OpenAIServingChat(
2637	mock_engine,
2638	models,
2639	response_role="assistant",
2640	online_renderer=online_renderer,
2641	chat_template=CHAT_TEMPLATE,
2642	chat_template_content_format="auto",
2643	request_logger=None,
2644	enable_auto_tools=False, # No tool parser
2645	)
2646
2647	tools = [
2648	{
2649	"type": "function",
2650	"function": {
2651	"name": "get_weather",
2652	"description": "Get the weather in a given location",
2653	"parameters": {
2654	"type": "object",
2655	"properties": {"location": {"type": "string"}},
2656	"required": ["location"],
2657	},
2658	},
2659	}
2660	]
2661
2662	# Test tool_choice="required" without tool_parser
2663	req_required = ChatCompletionRequest(
2664	model=MODEL_NAME,
2665	messages=[{"role": "user", "content": "What's the weather?"}],
2666	tools=tools,
2667	tool_choice="required",
2668	)
2669	response_required = await serving_chat.create_chat_completion(req_required)
2670	assert isinstance(response_required, ErrorResponse)
2671	assert "tool_choice" in response_required.error.message
2672	assert "--tool-call-parser" in response_required.error.message
2673
2674	# Test named tool_choice without tool_parser
2675	req_named = ChatCompletionRequest(
2676	model=MODEL_NAME,
2677	messages=[{"role": "user", "content": "What's the weather?"}],
2678	tools=tools,
2679	tool_choice={"type": "function", "function": {"name": "get_weather"}},
2680	)
2681	response_named = await serving_chat.create_chat_completion(req_named)
2682	assert isinstance(response_named, ErrorResponse)
2683	assert "tool_choice" in response_named.error.message
2684	assert "--tool-call-parser" in response_named.error.message
2685	# The function name should appear in a clean, readable form -
2686	# guards against leaking Pydantic's internal repr of the
2687	# ChatCompletionNamedToolChoiceParam/ChatCompletionNamedFunction
2688	# objects directly into the client-facing error message.
2689	assert "get_weather" in response_named.error.message
2690	assert "ChatCompletionNamedFunction" not in response_named.error.message
2691	assert "ChatCompletionNamedToolChoiceParam" not in response_named.error.message
2692
2693
2694	@pytest.mark.asyncio
2695	@pytest.mark.parametrize(
2696	("engine_finish_reason", "expected_finish_reason"),
2697	[("stop", "tool_calls"), ("length", "length")],
2698	)
2699	async def test_streaming_n_gt1_independent_tool_parsers(
2700	engine_finish_reason: str, expected_finish_reason: str
2701	):
2702	"""n>1 streaming must use independent parser instances
2703	and token-id histories per choice.
2704	"""
2705	mock_engine = MagicMock(spec=AsyncLLM)
2706	mock_engine.errored = False
2707	mock_engine.model_config = MockModelConfig()
2708	mock_engine.input_processor = MagicMock()
2709	mock_engine.renderer = _build_renderer(mock_engine.model_config)
2710
2711	models = OpenAIServingModels(
2712	engine_client=mock_engine,
2713	base_model_paths=BASE_MODEL_PATHS,
2714	)
2715	online_renderer = _build_online_renderer(mock_engine, models.registry)
2716
2717	serving_chat = OpenAIServingChat(
2718	mock_engine,
2719	models,
2720	response_role="assistant",
2721	online_renderer=online_renderer,
2722	chat_template=CHAT_TEMPLATE,
2723	chat_template_content_format="auto",
2724	request_logger=None,
2725	enable_auto_tools=True,
2726	tool_parser="hermes",
2727	)
2728
2729	tokenizer = get_tokenizer(MODEL_NAME)
2730
2731	tools = [
2732	{
2733	"type": "function",
2734	"function": {
2735	"name": "get_weather",
2736	"description": "Get weather",
2737	"parameters": {
2738	"type": "object",
2739	"properties": {"city": {"type": "string"}},
2740	"required": ["city"],
2741	},
2742	},
2743	}
2744	]
2745
2746	num_choices = 2
2747
2748	request = ChatCompletionRequest(
2749	model=MODEL_NAME,
2750	messages=[{"role": "user", "content": "test"}],
2751	n=num_choices,
2752	stream=True,
2753	tools=tools,
2754	tool_choice="auto",
2755	)
2756
2757	tool_call_text = (
2758	"<tool_call>\n"
2759	'{"name": "get_weather", "arguments": {"city": "Tokyo"}}\n'
2760	"</tool_call>"
2761	)
2762	all_token_ids = tokenizer.encode(tool_call_text, add_special_tokens=False)
2763
2764	# Compute proper delta text for each token so that concatenated deltas
2765	# reproduce the original string exactly.
2766	steps: list[tuple[str, int]] = []
2767	prev_decoded = ""
2768	for i, tid in enumerate(all_token_ids):
2769	decoded_so_far = tokenizer.decode(all_token_ids[: i + 1])
2770	delta = decoded_so_far[len(prev_decoded) :]
2771	steps.append((delta, tid))
2772	prev_decoded = decoded_so_far
2773
2774	async def result_generator():
2775	for delta_text, token_id in steps:
2776	yield RequestOutput(
2777	request_id="test-req",
2778	prompt="test",
2779	prompt_token_ids=[1, 2, 3],
2780	prompt_logprobs=None,
2781	outputs=[
2782	CompletionOutput(
2783	index=choice_idx,
2784	text=delta_text,
2785	token_ids=[token_id],
2786	cumulative_logprob=0.0,
2787	logprobs=None,
2788	)
2789	for choice_idx in range(num_choices)
2790	],
2791	finished=False,
2792	)
2793	# Final output with finish_reason
2794	yield RequestOutput(
2795	request_id="test-req",
2796	prompt="test",
2797	prompt_token_ids=[1, 2, 3],
2798	prompt_logprobs=None,
2799	outputs=[
2800	CompletionOutput(
2801	index=choice_idx,
2802	text="",
2803	token_ids=[],
2804	cumulative_logprob=0.0,
2805	logprobs=None,
2806	finish_reason=engine_finish_reason,
2807	)
2808	for choice_idx in range(num_choices)
2809	],
2810	finished=True,
2811	)
2812
2813	# Collect tool-call deltas and finish_reasons per choice from the SSE
2814	# stream.
2815	tc_deltas_by_choice: dict[int, list[dict]] = {i: [] for i in range(num_choices)}
2816	finish_reasons_by_choice: dict[int, list[str]] = {i: [] for i in range(num_choices)}
2817	async for chunk_str in serving_chat.chat_completion_stream_generator(
2818	request=request,
2819	result_generator=result_generator(),
2820	request_id="test-req",
2821	model_name=MODEL_NAME,
2822	conversation=[],
2823	tokenizer=tokenizer,
2824	request_metadata=RequestResponseMetadata(
2825	request_id="test-req",
2826	model_name=MODEL_NAME,
2827	),
2828	):
2829	if not chunk_str.strip() or "data: [DONE]" in chunk_str:
2830	continue
2831	if chunk_str.startswith("data: "):
2832	data = json.loads(chunk_str[6:].strip())
2833	for choice in data.get("choices", []):
2834	idx = choice["index"]
2835	delta = choice.get("delta", {})
2836	if delta.get("tool_calls"):
2837	for tc in delta["tool_calls"]:
2838	tc_deltas_by_choice[idx].append(tc)
2839	if choice.get("finish_reason") is not None:
2840	finish_reasons_by_choice[idx].append(choice["finish_reason"])
2841
2842	# Both choices must independently produce the correct tool call.
2843	for choice_idx in range(num_choices):
2844	deltas = tc_deltas_by_choice[choice_idx]
2845	assert len(deltas) > 0, (
2846	f"Choice {choice_idx}: expected tool-call deltas but got none"
2847	)
2848
2849	name = None
2850	args_buf = ""
2851	for tc in deltas:
2852	fn = tc.get("function", {})
2853	if fn.get("name"):
2854	name = fn["name"]
2855	if fn.get("arguments"):
2856	args_buf += fn["arguments"]
2857
2858	assert name == "get_weather", (
2859	f"Choice {choice_idx}: expected 'get_weather', got {name!r}"
2860	)
2861	parsed_args = json.loads(args_buf)
2862	assert parsed_args == {"city": "Tokyo"}, (
2863	f"Choice {choice_idx}: expected {{'city': 'Tokyo'}}, got {parsed_args}"
2864	)
2865
2866	reasons = finish_reasons_by_choice[choice_idx]
2867	assert len(reasons) == 1, (
2868	f"Choice {choice_idx}: expected exactly 1 finish_reason, got {reasons}"
2869	)
2870	assert reasons[0] == expected_finish_reason, (
2871	f"Choice {choice_idx}: expected finish_reason={expected_finish_reason!r}, "
2872	f"got '{reasons[0]}'"
2873	)
2874
2875
2876	def test_make_request_with_harmony_reuses_kv_transfer_prompt_token_ids():
2877	"""The Harmony reuse branch honors ids forwarded in kv_transfer_params.
2878
2879	A GPT-OSS server is impractical to stand up here, so this exercises the
2880	branch directly on a harmony-configured renderer.
2881	"""
2882	engine = MockEngine()
2883	engine.model_config.hf_config = MockHFConfig(model_type="gpt_oss")
2884	models = OpenAIServingModels(engine, BASE_MODEL_PATHS)
2885	online_renderer = _build_online_renderer(engine, models.registry)
2886	assert online_renderer.use_harmony
2887
2888	request = ChatCompletionRequest(
2889	model=MODEL_NAME,
2890	messages=[{"role": "user", "content": "hi"}],
2891	kv_transfer_params={
2892	"prompt_token_ids": [10, 20, 30],
2893	"do_remote_prefill": True,
2894	},
2895	)
2896	conversation, engine_inputs = online_renderer._make_request_with_harmony(request)
2897
2898	assert conversation == []
2899	assert len(engine_inputs) == 1
2900	engine_input = engine_inputs[0]
2901	assert engine_input["type"] == "token"
2902	assert engine_input["prompt_token_ids"] == [10, 20, 30]
2903	# The reuse key is consumed and other kv_transfer_params are preserved.
2904	assert request.kv_transfer_params == {"do_remote_prefill": True}
2905
2906
2907	@pytest.mark.parametrize("ids", [[-1], [1.5]])
2908	def test_make_request_with_harmony_rejects_invalid_kv_transfer_prompt_token_ids(ids):
2909	engine = MockEngine()
2910	engine.model_config.hf_config = MockHFConfig(model_type="gpt_oss")
2911	models = OpenAIServingModels(engine, BASE_MODEL_PATHS)
2912	online_renderer = _build_online_renderer(engine, models.registry)
2913
2914	request = ChatCompletionRequest(
2915	model=MODEL_NAME,
2916	messages=[{"role": "user", "content": "hi"}],
2917	kv_transfer_params={"prompt_token_ids": ids},
2918	)
2919	with pytest.raises(VLLMValidationError, match="non-negative integers"):
2920	online_renderer._make_request_with_harmony(request)
2921
2922
2923	@pytest.mark.asyncio
2924	@pytest.mark.parametrize("ids", [[], [-1], [1.5], [1.0], ["1"], [True], "abc", 5])
2925	async def test_chat_kv_transfer_prompt_token_ids_rejects_invalid_ids(ids):
2926	"""``kv_transfer_params`` is untyped, so the ids are checked on reuse."""
2927	serving_chat = _build_serving_chat(_build_mock_engine())
2928
2929	request = ChatCompletionRequest(
2930	model=MODEL_NAME,
2931	messages=[{"role": "user", "content": "hi"}],
2932	kv_transfer_params={"prompt_token_ids": ids},
2933	)
2934	with pytest.raises(VLLMValidationError, match="non-negative integers"):
2935	await serving_chat.render_chat_request(request)
2936
2937
2938	@pytest.mark.asyncio
2939	@pytest.mark.parametrize(
2940	("ids", "stream", "match"),
2941	[
2942	([1.5], False, "non-negative integers"),
2943	([999999], True, r"out of vocabulary.*kv_transfer_params\.prompt_token_ids"),
2944	],
2945	)
2946	async def test_chat_kv_transfer_prompt_token_ids_rejection_notifies_kv_connector(
2947	ids, stream, match
2948	):
2949	"""A render-time 400 on a decode request notifies the KV connector."""
2950	mock_engine = _build_mock_engine()
2951	mock_engine.notify_kv_transfer_request_rejected = AsyncMock()
2952	serving_chat = _build_serving_chat(mock_engine)
2953	serving_chat.has_kv_connector = True
2954
2955	request = ChatCompletionRequest(
2956	model=MODEL_NAME,
2957	messages=[{"role": "user", "content": "hi"}],
2958	stream=stream,
2959	kv_transfer_params={"prompt_token_ids": ids, "do_remote_prefill": True},
2960	)
2961	with pytest.raises(VLLMValidationError, match=match):
2962	await serving_chat.create_chat_completion(request)
2963
2964	mock_engine.notify_kv_transfer_request_rejected.assert_awaited_once_with(
2965	request.request_id, {"do_remote_prefill": True}, data_parallel_rank=None
2966	)
2967
2968
2969	@pytest.mark.asyncio
2970	@pytest.mark.parametrize("ids", [[10, 20, 30], [1.5]])
2971	async def test_chat_kv_transfer_prompt_token_ids_ignored_with_echo(ids):
2972	"""``echo`` needs the rendered conversation, so the ids are not used."""
2973	serving_chat = _build_serving_chat(_build_mock_engine())
2974
2975	request = ChatCompletionRequest(
2976	model=MODEL_NAME,
2977	messages=[{"role": "user", "content": "hi"}],
2978	kv_transfer_params={"prompt_token_ids": ids, "do_remote_prefill": True},
2979	echo=True,
2980	)
2981	result = await serving_chat.render_chat_request(request)
2982	assert not isinstance(result, ErrorResponse)
2983
2984	conversation, engine_inputs = result
2985	assert [msg["role"] for msg in conversation] == ["user"]
2986	assert engine_inputs[0]["prompt_token_ids"] != ids
2987	assert request.kv_transfer_params == {"do_remote_prefill": True}
2988
2989
2990	@pytest.mark.parametrize(
2991	"content",
2992	[
2993	[
2994	{"type": "text", "text": "what is in this image?"},
2995	{"type": "image_url", "image_url": {"url": "https://example.com/a.png"}},
2996	],
2997	# A part written without a "type" is identified by its media key.
2998	[{"image_url": "https://example.com/a.png"}],
2999	# A media key counts even when the part claims to be text, which is
3000	# how chat_utils reads a part carrying a "uuid".
3001	[
3002	{
3003	"type": "text",
3004	"text": "look",
3005	"uuid": "u1",
3006	"image_url": "https://example.com/a.png",
3007	}
3008	],
3009	[{"type": "hologram", "hologram": "https://a/b.holo"}],
3010	[{"type": ["image_url"]}],
3011	],
3012	)
3013	def test_chat_kv_transfer_prompt_token_ids_ignored_with_non_text_content(content):
3014	"""The ids are dropped so that ``messages`` is rendered with its media."""
3015	request = ChatCompletionRequest(
3016	model=MODEL_NAME,
3017	messages=[{"role": "user", "content": content}],
3018	kv_transfer_params={
3019	"prompt_token_ids": [10, 20, 30],
3020	"do_remote_prefill": True,
3021	},
3022	)
3023	assert request.kv_transfer_params == {"do_remote_prefill": True}
3024
3025
3026	def test_chat_kv_transfer_prompt_token_ids_allows_text_only_parts():
3027	"""Text-bearing part types are not multimodal, so the ids are kept."""
3028	request = ChatCompletionRequest(
3029	model=MODEL_NAME,
3030	messages=[
3031	{"role": "user", "content": [{"type": "text", "text": "hi"}]},
3032	{
3033	"role": "assistant",
3034	"content": [
3035	{"type": "thinking", "thinking": "let me think"},
3036	{"type": "refusal", "refusal": "no"},
3037	],
3038	},
3039	{"role": "user", "content": "plain string"},
3040	],
3041	kv_transfer_params={"prompt_token_ids": [10, 20, 30]},
3042	)
3043	assert request.kv_transfer_params == {"prompt_token_ids": [10, 20, 30]}