@tduncan / Nvidia-Optimization-Stack-VLLM
Code Commits Issues Pulls Cloud Agent CI/CD
main
main
py 178 lines 5.56 KB
Raw
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 """E2E tests for ``include_reasoning`` with non-Harmony reasoning models.
5
6 Verifies that reasoning content is included by default and suppressed
7 when ``include_reasoning=False``, for both streaming and non-streaming
8 Chat Completions, while reasoning tokens remain represented in usage.
9 """
10
11 import openai
12 import pytest
13 import pytest_asyncio
14
15 from tests.utils import RemoteOpenAIServer
16
17 MODEL_NAME = "Qwen/Qwen3-0.6B"
18 MESSAGES = [{"role": "user", "content": "What is 1+1? Be concise."}]
19
20
21 @pytest.fixture(scope="module")
22 def server():
23 args = [
24 "--reasoning-parser",
25 "qwen3",
26 "--max-model-len",
27 "2048",
28 "--enforce-eager",
29 "--gpu-memory-utilization",
30 "0.4",
31 ]
32 with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
33 yield remote_server
34
35
36 @pytest_asyncio.fixture
37 async def client(server):
38 async with server.get_async_client() as async_client:
39 yield async_client
40
41
42 @pytest.mark.asyncio
43 async def test_include_reasoning_true_non_streaming(client: openai.AsyncOpenAI):
44 """Default: reasoning content appears in non-streaming response."""
45 response = await client.chat.completions.create(
46 model=MODEL_NAME,
47 messages=MESSAGES,
48 max_tokens=512,
49 temperature=0,
50 extra_body={"include_reasoning": True},
51 )
52
53 msg = response.choices[0].message
54 reasoning = getattr(msg, "reasoning", None) or getattr(
55 msg, "reasoning_content", None
56 )
57 assert reasoning, "Expected reasoning content when include_reasoning=True"
58 assert msg.content, "Expected content in response"
59
60
61 @pytest.mark.asyncio
62 async def test_include_reasoning_false_non_streaming(client: openai.AsyncOpenAI):
63 """Reasoning content is suppressed when include_reasoning=False."""
64 response = await client.chat.completions.create(
65 model=MODEL_NAME,
66 messages=MESSAGES,
67 max_tokens=512,
68 temperature=0,
69 extra_body={"include_reasoning": False},
70 )
71
72 msg = response.choices[0].message
73 reasoning = getattr(msg, "reasoning", None) or getattr(
74 msg, "reasoning_content", None
75 )
76 assert not reasoning, (
77 f"Expected no reasoning when include_reasoning=False, got: {reasoning}"
78 )
79 assert msg.content, "Expected content in response even without reasoning"
80
81
82 @pytest.mark.asyncio
83 async def test_reasoning_tokens_in_usage(client: openai.AsyncOpenAI):
84 """Hidden reasoning still contributes to terminal usage details."""
85 response = await client.chat.completions.create(
86 model=MODEL_NAME,
87 messages=MESSAGES,
88 max_tokens=64,
89 extra_body={"include_reasoning": False},
90 )
91
92 assert response.usage is not None
93 details = response.usage.completion_tokens_details
94 assert details is not None
95 assert details.reasoning_tokens > 0
96
97
98 @pytest.mark.asyncio
99 async def test_include_reasoning_true_streaming(client: openai.AsyncOpenAI):
100 """Default: reasoning deltas appear in streaming response."""
101 stream = await client.chat.completions.create(
102 model=MODEL_NAME,
103 messages=MESSAGES,
104 max_tokens=512,
105 temperature=0,
106 stream=True,
107 extra_body={"include_reasoning": True},
108 )
109
110 reasoning_parts = []
111 content_parts = []
112 async for chunk in stream:
113 delta = chunk.choices[0].delta if chunk.choices else None
114 if delta:
115 r = getattr(delta, "reasoning", None) or getattr(
116 delta, "reasoning_content", None
117 )
118 if r:
119 reasoning_parts.append(r)
120 if delta.content:
121 content_parts.append(delta.content)
122
123 reasoning_text = "".join(reasoning_parts)
124 content_text = "".join(content_parts)
125
126 assert reasoning_text, "Expected reasoning deltas when include_reasoning=True"
127 assert content_text, "Expected content deltas in streaming response"
128
129
130 @pytest.mark.asyncio
131 async def test_include_reasoning_false_streaming(client: openai.AsyncOpenAI):
132 """Reasoning deltas are suppressed in streaming when include_reasoning=False."""
133 stream = await client.chat.completions.create(
134 model=MODEL_NAME,
135 messages=MESSAGES,
136 max_tokens=512,
137 temperature=0,
138 stream=True,
139 extra_body={"include_reasoning": False},
140 )
141
142 reasoning_parts = []
143 content_parts = []
144 async for chunk in stream:
145 delta = chunk.choices[0].delta if chunk.choices else None
146 if delta:
147 r = getattr(delta, "reasoning", None) or getattr(
148 delta, "reasoning_content", None
149 )
150 if r:
151 reasoning_parts.append(r)
152 if delta.content:
153 content_parts.append(delta.content)
154
155 reasoning_text = "".join(reasoning_parts)
156 content_text = "".join(content_parts)
157
158 assert not reasoning_text, (
159 f"Expected no reasoning deltas when include_reasoning=False, "
160 f"got: {reasoning_text[:100]}"
161 )
162 assert content_text, "Expected content deltas even without reasoning"
163
164
165 @pytest.mark.asyncio
166 async def test_default_includes_reasoning(client: openai.AsyncOpenAI):
167 """Without specifying include_reasoning, reasoning appears (default=True)."""
168 response = await client.chat.completions.create(
169 model=MODEL_NAME,
170 messages=MESSAGES,
171 max_tokens=200,
172 )
173
174 msg = response.choices[0].message
175 reasoning = getattr(msg, "reasoning", None) or getattr(
176 msg, "reasoning_content", None
177 )
178 assert reasoning, "Expected reasoning content by default"