| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import importlib.util |
| 5 | import json |
| 6 | import logging |
| 7 | |
| 8 | import pytest |
| 9 | import pytest_asyncio |
| 10 | from openai import OpenAI |
| 11 | |
| 12 | from tests.utils import RemoteOpenAIServer |
| 13 | |
| 14 | from .conftest import ( |
| 15 | BASE_TEST_ENV, |
| 16 | has_output_type, |
| 17 | log_response_diagnostics, |
| 18 | retry_for_tool_call, |
| 19 | ) |
| 20 | |
| 21 | logger = logging.getLogger(__name__) |
| 22 | |
| 23 | MODEL_NAME = "Qwen/Qwen3-8B" |
| 24 | |
| 25 | _PYTHON_TOOL_INSTRUCTION = ( |
| 26 | "You must use the Python tool to execute code. " |
| 27 | "Never simulate execution. You must print the final answer." |
| 28 | ) |
| 29 | |
| 30 | |
| 31 | @pytest.fixture(scope="module") |
| 32 | def server(): |
| 33 | assert importlib.util.find_spec("gpt_oss") is not None, ( |
| 34 | "Harmony tests require gpt_oss package to be installed" |
| 35 | ) |
| 36 | |
| 37 | args = [ |
| 38 | "--reasoning-parser", |
| 39 | "qwen3", |
| 40 | "--max_model_len", |
| 41 | "6000", |
| 42 | "--structured-outputs-config.backend", |
| 43 | "xgrammar", |
| 44 | "--enable-auto-tool-choice", |
| 45 | "--tool-call-parser", |
| 46 | "hermes", |
| 47 | "--tool-server", |
| 48 | "demo", |
| 49 | ] |
| 50 | env_dict = { |
| 51 | **BASE_TEST_ENV, |
| 52 | "VLLM_ENABLE_RESPONSES_API_STORE": "1", |
| 53 | "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT": "1", |
| 54 | "PYTHON_EXECUTION_BACKEND": "dangerously_use_uv", |
| 55 | } |
| 56 | with RemoteOpenAIServer(MODEL_NAME, args, env_dict=env_dict) as remote_server: |
| 57 | yield remote_server |
| 58 | |
| 59 | |
| 60 | @pytest_asyncio.fixture |
| 61 | async def client(server): |
| 62 | async with server.get_async_client() as async_client: |
| 63 | yield async_client |
| 64 | |
| 65 | |
| 66 | @pytest.mark.asyncio |
| 67 | @pytest.mark.parametrize("model_name", [MODEL_NAME]) |
| 68 | async def test_basic(client: OpenAI, model_name: str): |
| 69 | response = await client.responses.create( |
| 70 | model=model_name, |
| 71 | input="What is 123 * 456? Answer with only the number.", |
| 72 | temperature=0.0, |
| 73 | ) |
| 74 | assert response is not None |
| 75 | print("response: ", response) |
| 76 | assert response.status == "completed" |
| 77 | assert response.incomplete_details is None |
| 78 | |
| 79 | |
| 80 | @pytest.mark.asyncio |
| 81 | @pytest.mark.parametrize("model_name", [MODEL_NAME]) |
| 82 | async def test_reasoning_and_function_items(client: OpenAI, model_name: str): |
| 83 | response = await client.responses.create( |
| 84 | model=model_name, |
| 85 | input=[ |
| 86 | {"type": "message", "content": "Hello.", "role": "user"}, |
| 87 | { |
| 88 | "type": "reasoning", |
| 89 | "id": "lol", |
| 90 | "content": [ |
| 91 | { |
| 92 | "type": "reasoning_text", |
| 93 | "text": "We need to respond: greeting.", |
| 94 | } |
| 95 | ], |
| 96 | "summary": [], |
| 97 | }, |
| 98 | { |
| 99 | "arguments": '{"location": "Paris", "unit": "celsius"}', |
| 100 | "call_id": "call_5f7b38f3b81e4b8380fd0ba74f3ca3ab", |
| 101 | "name": "get_weather", |
| 102 | "type": "function_call", |
| 103 | "id": "fc_4fe5d6fc5b6c4d6fa5f24cc80aa27f78", |
| 104 | "status": "completed", |
| 105 | }, |
| 106 | { |
| 107 | "call_id": "call_5f7b38f3b81e4b8380fd0ba74f3ca3ab", |
| 108 | "id": "fc_4fe5d6fc5b6c4d6fa5f24cc80aa27f78", |
| 109 | "output": "The weather in Paris is 20 Celsius", |
| 110 | "status": "completed", |
| 111 | "type": "function_call_output", |
| 112 | }, |
| 113 | ], |
| 114 | temperature=0.0, |
| 115 | ) |
| 116 | assert response is not None |
| 117 | assert response.status == "completed" |
| 118 | |
| 119 | output_types = [getattr(o, "type", None) for o in response.output] |
| 120 | assert "reasoning" in output_types, ( |
| 121 | f"Expected reasoning in output, got: {output_types}" |
| 122 | ) |
| 123 | assert "message" in output_types, f"Expected message in output, got: {output_types}" |
| 124 | |
| 125 | msg = next(o for o in response.output if o.type == "message") |
| 126 | assert type(msg.content[0].text) is str |
| 127 | |
| 128 | |
| 129 | def get_horoscope(sign): |
| 130 | return f"{sign}: Next Tuesday you will befriend a baby otter." |
| 131 | |
| 132 | |
| 133 | def call_function(name, args): |
| 134 | logger.info("Calling function %s with args %s", name, args) |
| 135 | if name == "get_horoscope": |
| 136 | return get_horoscope(**args) |
| 137 | raise ValueError(f"Unknown function: {name}") |
| 138 | |
| 139 | |
| 140 | @pytest.mark.asyncio |
| 141 | @pytest.mark.parametrize("model_name", [MODEL_NAME]) |
| 142 | async def test_function_call_first_turn(client: OpenAI, model_name: str): |
| 143 | tools = [ |
| 144 | { |
| 145 | "type": "function", |
| 146 | "name": "get_horoscope", |
| 147 | "description": "Get today's horoscope for an astrological sign.", |
| 148 | "parameters": { |
| 149 | "type": "object", |
| 150 | "properties": { |
| 151 | "sign": {"type": "string"}, |
| 152 | }, |
| 153 | "required": ["sign"], |
| 154 | "additionalProperties": False, |
| 155 | }, |
| 156 | "strict": True, |
| 157 | } |
| 158 | ] |
| 159 | |
| 160 | response = await retry_for_tool_call( |
| 161 | client, |
| 162 | model=model_name, |
| 163 | expected_tool_type="function_call", |
| 164 | input="What is the horoscope for Aquarius today?", |
| 165 | tools=tools, |
| 166 | temperature=0.0, |
| 167 | ) |
| 168 | assert response is not None |
| 169 | assert response.status == "completed" |
| 170 | |
| 171 | output_types = [getattr(o, "type", None) for o in response.output] |
| 172 | assert "reasoning" in output_types, ( |
| 173 | f"Expected reasoning in output, got: {output_types}" |
| 174 | ) |
| 175 | assert has_output_type(response, "function_call"), ( |
| 176 | f"Expected function_call in output, got: {output_types}" |
| 177 | ) |
| 178 | |
| 179 | function_call = next(o for o in response.output if o.type == "function_call") |
| 180 | assert function_call.name == "get_horoscope" |
| 181 | assert function_call.call_id is not None |
| 182 | |
| 183 | args = json.loads(function_call.arguments) |
| 184 | assert "sign" in args |
| 185 | |
| 186 | |
| 187 | @pytest.mark.asyncio |
| 188 | @pytest.mark.parametrize("model_name", [MODEL_NAME]) |
| 189 | @pytest.mark.xfail( |
| 190 | reason=( |
| 191 | "MCP tools are not properly supported: tool name parameters " |
| 192 | "are not extracted for prompt rendering." |
| 193 | ), |
| 194 | ) |
| 195 | async def test_mcp_tool_call(client: OpenAI, model_name: str): |
| 196 | """MCP tool calling with code_interpreter. |
| 197 | |
| 198 | The model may make one or more tool calls before producing a final |
| 199 | message. We validate server invariants (mcp_call items have correct |
| 200 | fields) with hard assertions. Output indices are never hardcoded |
| 201 | since the model can produce multiple tool-call rounds. |
| 202 | """ |
| 203 | # MCP + container init + code execution can be slow |
| 204 | client_with_timeout = client.with_options(timeout=client.timeout * 3) |
| 205 | |
| 206 | response = await retry_for_tool_call( |
| 207 | client_with_timeout, |
| 208 | model=model_name, |
| 209 | expected_tool_type="mcp_call", |
| 210 | input=( |
| 211 | "What is 123 * 456? Use python to calculate the result. " |
| 212 | "Print the result with print()." |
| 213 | ), |
| 214 | tools=[{"type": "code_interpreter", "container": {"type": "auto"}}], |
| 215 | instructions=_PYTHON_TOOL_INSTRUCTION, |
| 216 | temperature=0.0, |
| 217 | extra_body={"enable_response_messages": True}, |
| 218 | ) |
| 219 | |
| 220 | assert response is not None |
| 221 | |
| 222 | output_types = [getattr(o, "type", None) for o in response.output] |
| 223 | log_response_diagnostics(response, label="test_mcp_tool_call") |
| 224 | |
| 225 | assert response.status == "completed", ( |
| 226 | f"Response status={response.status} " |
| 227 | f"(details={getattr(response, 'incomplete_details', None)}). " |
| 228 | f"Output types: {output_types}." |
| 229 | ) |
| 230 | |
| 231 | assert "reasoning" in output_types, ( |
| 232 | f"Expected reasoning in output, got: {output_types}" |
| 233 | ) |
| 234 | assert "mcp_call" in output_types, ( |
| 235 | f"Expected mcp_call in output, got: {output_types}" |
| 236 | ) |
| 237 | |
| 238 | # Every mcp_call item must have well-typed fields |
| 239 | for item in response.output: |
| 240 | if getattr(item, "type", None) == "mcp_call": |
| 241 | assert type(item.arguments) is str, ( |
| 242 | f"mcp_call.arguments should be str, got {type(item.arguments)}" |
| 243 | ) |
| 244 | assert type(item.output) is str, ( |
| 245 | f"mcp_call.output should be str, got {type(item.output)}" |
| 246 | ) |
| 247 | |
| 248 | # The model may make 1+ tool-call rounds but must still produce |
| 249 | # a final message for a trivial calculation like 123 * 456. |
| 250 | message_outputs = [ |
| 251 | o for o in response.output if getattr(o, "type", None) == "message" |
| 252 | ] |
| 253 | assert message_outputs, ( |
| 254 | f"Model did not produce a final message. Output types: {output_types}" |
| 255 | ) |
| 256 | |
| 257 | final_message = message_outputs[-1] |
| 258 | assert any(s in final_message.content[0].text for s in ("56088", "56,088")), ( |
| 259 | f"Expected 56088 in final message, got: {final_message.content[0].text!r}" |
| 260 | ) |
| 261 | |
| 262 | # Validate raw input_messages / output_messages |
| 263 | assert len(response.input_messages) >= 1, "Expected at least 1 input message" |
| 264 | assert len(response.output_messages) >= 1, "Expected at least 1 output message" |
| 265 | assert any( |
| 266 | any(s in str(msg) for s in ("56088", "56,088")) |
| 267 | for msg in response.output_messages |
| 268 | ), ( |
| 269 | f"Expected 56088 in at least one output_message, " |
| 270 | f"got {len(response.output_messages)} messages" |
| 271 | ) |
| 272 | |
| 273 | |
| 274 | @pytest.mark.asyncio |
| 275 | @pytest.mark.parametrize("model_name", [MODEL_NAME]) |
| 276 | async def test_max_tokens(client: OpenAI, model_name: str): |
| 277 | response = await client.responses.create( |
| 278 | model=model_name, |
| 279 | input="What is the first paragraph of Moby Dick?", |
| 280 | reasoning={"effort": "low"}, |
| 281 | max_output_tokens=30, |
| 282 | temperature=0.0, |
| 283 | ) |
| 284 | assert response is not None |
| 285 | assert response.status == "incomplete" |
| 286 | assert response.incomplete_details.reason == "max_output_tokens" |