| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import pytest |
| 5 | import pytest_asyncio |
| 6 | from openai import OpenAI |
| 7 | |
| 8 | from tests.utils import RemoteOpenAIServer |
| 9 | |
| 10 | from .conftest import validate_streaming_event_stack |
| 11 | |
| 12 | MODEL_NAME = "Qwen/Qwen3-8B" |
| 13 | |
| 14 | |
| 15 | @pytest.fixture(scope="module") |
| 16 | def server(): |
| 17 | from .conftest import BASE_TEST_ENV |
| 18 | |
| 19 | args = ["--reasoning-parser", "qwen3", "--max_model_len", "5000"] |
| 20 | env_dict = { |
| 21 | **BASE_TEST_ENV, |
| 22 | "VLLM_ENABLE_RESPONSES_API_STORE": "1", |
| 23 | # uncomment for tool calling |
| 24 | # PYTHON_EXECUTION_BACKEND: "dangerously_use_uv", |
| 25 | } |
| 26 | with RemoteOpenAIServer(MODEL_NAME, args, env_dict=env_dict) as remote_server: |
| 27 | yield remote_server |
| 28 | |
| 29 | |
| 30 | @pytest_asyncio.fixture |
| 31 | async def client(server): |
| 32 | async with server.get_async_client() as async_client: |
| 33 | yield async_client |
| 34 | |
| 35 | |
| 36 | @pytest.mark.asyncio |
| 37 | @pytest.mark.parametrize("model_name", [MODEL_NAME]) |
| 38 | async def test_basic(client: OpenAI, model_name: str): |
| 39 | response = await client.responses.create( |
| 40 | model=model_name, |
| 41 | input="What is 123 * 456?", |
| 42 | reasoning={"effort": "none"}, |
| 43 | ) |
| 44 | assert response is not None |
| 45 | print("response: ", response) |
| 46 | assert response.status == "completed" |
| 47 | assert response.incomplete_details is None |
| 48 | |
| 49 | |
| 50 | @pytest.mark.asyncio |
| 51 | @pytest.mark.parametrize("model_name", [MODEL_NAME]) |
| 52 | async def test_enable_response_messages(client: OpenAI, model_name: str): |
| 53 | response = await client.responses.create( |
| 54 | model=model_name, |
| 55 | input="Hello?", |
| 56 | extra_body={"enable_response_messages": True}, |
| 57 | ) |
| 58 | assert response.status == "completed" |
| 59 | assert response.input_messages[0]["type"] == "raw_message_tokens" |
| 60 | assert type(response.input_messages[0]["message"]) is str |
| 61 | assert len(response.input_messages[0]["message"]) > 10 |
| 62 | assert type(response.input_messages[0]["tokens"][0]) is int |
| 63 | assert type(response.output_messages[0]["message"]) is str |
| 64 | assert len(response.output_messages[0]["message"]) > 10 |
| 65 | assert type(response.output_messages[0]["tokens"][0]) is int |
| 66 | |
| 67 | |
| 68 | @pytest.mark.asyncio |
| 69 | @pytest.mark.parametrize("model_name", [MODEL_NAME]) |
| 70 | async def test_reasoning_item(client: OpenAI, model_name: str): |
| 71 | response = await client.responses.create( |
| 72 | model=model_name, |
| 73 | input=[ |
| 74 | {"type": "message", "content": "Hello.", "role": "user"}, |
| 75 | { |
| 76 | "type": "reasoning", |
| 77 | "id": "lol", |
| 78 | "content": [ |
| 79 | { |
| 80 | "type": "reasoning_text", |
| 81 | "text": "We need to respond: greeting.", |
| 82 | } |
| 83 | ], |
| 84 | "summary": [], |
| 85 | }, |
| 86 | ], |
| 87 | temperature=0.0, |
| 88 | ) |
| 89 | assert response is not None |
| 90 | assert response.status == "completed" |
| 91 | # make sure we get a reasoning and text output |
| 92 | assert response.output[0].type == "reasoning" |
| 93 | assert response.output[1].type == "message" |
| 94 | assert type(response.output[1].content[0].text) is str |
| 95 | |
| 96 | |
| 97 | @pytest.mark.asyncio |
| 98 | @pytest.mark.parametrize("model_name", [MODEL_NAME]) |
| 99 | async def test_streaming_output_consistency(client: OpenAI, model_name: str): |
| 100 | """Test that streaming delta text matches the final response output_text. |
| 101 | |
| 102 | This test verifies that when using streaming mode: |
| 103 | 1. The concatenated text from all 'response.output_text.delta' events |
| 104 | 2. Matches the 'output_text' in the final 'response.completed' event |
| 105 | """ |
| 106 | response = await client.responses.create( |
| 107 | model=model_name, |
| 108 | input="Say hello in one sentence.", |
| 109 | stream=True, |
| 110 | ) |
| 111 | |
| 112 | events = [] |
| 113 | async for event in response: |
| 114 | events.append(event) |
| 115 | |
| 116 | assert len(events) > 0 |
| 117 | |
| 118 | # Concatenate all delta text from streaming events |
| 119 | streaming_text = "".join( |
| 120 | event.delta for event in events if event.type == "response.output_text.delta" |
| 121 | ) |
| 122 | |
| 123 | # Get the final response from the last event |
| 124 | response_completed_event = events[-1] |
| 125 | assert response_completed_event.type == "response.completed" |
| 126 | assert response_completed_event.response.status == "completed" |
| 127 | |
| 128 | # Get output_text from the final response |
| 129 | final_output_text = response_completed_event.response.output_text |
| 130 | |
| 131 | # Verify final response has output |
| 132 | assert len(response_completed_event.response.output) > 0 |
| 133 | |
| 134 | # Verify streaming text matches final output_text |
| 135 | assert streaming_text == final_output_text, ( |
| 136 | f"Streaming text does not match final output_text.\n" |
| 137 | f"Streaming: {streaming_text!r}\n" |
| 138 | f"Final: {final_output_text!r}" |
| 139 | ) |
| 140 | |
| 141 | |
| 142 | @pytest.mark.asyncio |
| 143 | @pytest.mark.parametrize("model_name", [MODEL_NAME]) |
| 144 | async def test_streaming_logprobs(client: OpenAI, model_name: str): |
| 145 | """Test that streaming with logprobs returns valid logprob data on |
| 146 | output_text.delta events and that top_logprobs has the requested count.""" |
| 147 | response = await client.responses.create( |
| 148 | model=model_name, |
| 149 | input="Say hello.", |
| 150 | stream=True, |
| 151 | top_logprobs=3, |
| 152 | include=["message.output_text.logprobs"], |
| 153 | ) |
| 154 | |
| 155 | events = [] |
| 156 | async for event in response: |
| 157 | events.append(event) |
| 158 | |
| 159 | assert len(events) > 0 |
| 160 | |
| 161 | # Collect all output_text.delta events that carry logprobs |
| 162 | text_delta_events = [e for e in events if e.type == "response.output_text.delta"] |
| 163 | assert len(text_delta_events) > 0, "Expected at least one text delta event" |
| 164 | |
| 165 | for delta_event in text_delta_events: |
| 166 | logprobs = delta_event.logprobs |
| 167 | assert logprobs is not None, "logprobs should be present on text delta events" |
| 168 | assert len(logprobs) > 0, "logprobs list should not be empty" |
| 169 | for lp in logprobs: |
| 170 | # Each logprob entry must have a token and a logprob value |
| 171 | assert lp.token is not None |
| 172 | assert isinstance(lp.logprob, float) |
| 173 | assert lp.logprob <= 0.0, f"logprob should be <= 0, got {lp.logprob}" |
| 174 | # top_logprobs should have up to 3 entries |
| 175 | assert lp.top_logprobs is not None |
| 176 | assert len(lp.top_logprobs) <= 3 |
| 177 | for tl in lp.top_logprobs: |
| 178 | assert tl.token is not None |
| 179 | assert isinstance(tl.logprob, float) |
| 180 | |
| 181 | # Verify that top_logprobs are actually populated, not always empty |
| 182 | all_top_logprobs = [ |
| 183 | tl for e in text_delta_events for lp in e.logprobs for tl in lp.top_logprobs |
| 184 | ] |
| 185 | assert len(all_top_logprobs) > 0, ( |
| 186 | "Expected at least one top_logprobs entry across all delta events" |
| 187 | ) |
| 188 | |
| 189 | # Verify the completed event still has valid output |
| 190 | completed = events[-1] |
| 191 | assert completed.type == "response.completed" |
| 192 | assert completed.response.status == "completed" |
| 193 | |
| 194 | |
| 195 | @pytest.mark.asyncio |
| 196 | @pytest.mark.parametrize("model_name", [MODEL_NAME]) |
| 197 | async def test_streaming_reasoning_tokens_e2e(client: OpenAI, model_name: str): |
| 198 | """Verify final usage includes reasoning_tokens in streaming mode.""" |
| 199 | response = await client.responses.create( |
| 200 | model=model_name, |
| 201 | input="Compute 17 * 19 and explain briefly.", |
| 202 | reasoning={"effort": "low"}, |
| 203 | temperature=0.0, |
| 204 | stream=True, |
| 205 | ) |
| 206 | |
| 207 | completed_event = None |
| 208 | async for event in response: |
| 209 | if event.type == "response.completed": |
| 210 | completed_event = event |
| 211 | |
| 212 | assert completed_event is not None |
| 213 | assert completed_event.response.status == "completed" |
| 214 | assert completed_event.response.usage is not None |
| 215 | assert completed_event.response.usage.output_tokens_details is not None |
| 216 | assert completed_event.response.usage.output_tokens_details.reasoning_tokens > 0, ( |
| 217 | "Expected reasoning_tokens > 0 for streamed Qwen3 response." |
| 218 | ) |
| 219 | |
| 220 | |
| 221 | @pytest.mark.asyncio |
| 222 | @pytest.mark.parametrize("model_name", [MODEL_NAME]) |
| 223 | async def test_non_streaming_reasoning_tokens_e2e(client: OpenAI, model_name: str): |
| 224 | """Verify usage includes reasoning_tokens in non-streaming mode.""" |
| 225 | response = await client.responses.create( |
| 226 | model=model_name, |
| 227 | input="Compute 23 * 17 and explain briefly.", |
| 228 | reasoning={"effort": "low"}, |
| 229 | temperature=0.0, |
| 230 | stream=False, |
| 231 | ) |
| 232 | |
| 233 | assert response is not None |
| 234 | assert response.status == "completed" |
| 235 | assert response.usage is not None |
| 236 | assert response.usage.output_tokens_details is not None |
| 237 | assert response.usage.output_tokens_details.reasoning_tokens > 0, ( |
| 238 | "Expected reasoning_tokens > 0 for non-streamed Qwen3 response." |
| 239 | ) |
| 240 | |
| 241 | |
| 242 | @pytest.mark.asyncio |
| 243 | @pytest.mark.parametrize("model_name", [MODEL_NAME]) |
| 244 | async def test_max_tokens(client: OpenAI, model_name: str): |
| 245 | response = await client.responses.create( |
| 246 | model=model_name, |
| 247 | input="What is the first paragraph of Moby Dick?", |
| 248 | reasoning={"effort": "low"}, |
| 249 | max_output_tokens=30, |
| 250 | ) |
| 251 | assert response is not None |
| 252 | assert response.status == "incomplete" |
| 253 | assert response.incomplete_details.reason == "max_output_tokens" |
| 254 | |
| 255 | |
| 256 | @pytest.mark.asyncio |
| 257 | @pytest.mark.parametrize("model_name", [MODEL_NAME]) |
| 258 | async def test_extra_sampling_params(client: OpenAI, model_name: str): |
| 259 | """Test that extra sampling parameters are accepted and work.""" |
| 260 | # Test with multiple sampling parameters - just verify they're accepted |
| 261 | response = await client.responses.create( |
| 262 | model=model_name, |
| 263 | input="Write a short sentence", |
| 264 | max_output_tokens=50, |
| 265 | temperature=0.7, |
| 266 | top_p=0.9, |
| 267 | extra_body={ |
| 268 | "top_k": 40, |
| 269 | "repetition_penalty": 1.2, |
| 270 | "seed": 42, |
| 271 | }, |
| 272 | ) |
| 273 | |
| 274 | # Verify request succeeded and parameters were accepted |
| 275 | assert response.status in ["completed", "incomplete"] |
| 276 | assert len(response.output) > 0 |
| 277 | assert response.output[0].content[0].text # Has text output |
| 278 | |
| 279 | |
| 280 | @pytest.mark.asyncio |
| 281 | @pytest.mark.parametrize("model_name", [MODEL_NAME]) |
| 282 | async def test_streaming_types( |
| 283 | pairs_of_event_types: dict[str, str], client: OpenAI, model_name: str |
| 284 | ): |
| 285 | stream = await client.responses.create( |
| 286 | model=model_name, |
| 287 | input="tell me a story about a cat in 20 words", |
| 288 | reasoning={"effort": "low"}, |
| 289 | tools=[], |
| 290 | stream=True, |
| 291 | background=False, |
| 292 | ) |
| 293 | events = [] |
| 294 | async for event in stream: |
| 295 | events.append(event) |
| 296 | |
| 297 | validate_streaming_event_stack(events, pairs_of_event_types) |