| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import json |
| 5 | |
| 6 | import jsonschema |
| 7 | import openai |
| 8 | import pytest |
| 9 | import pytest_asyncio |
| 10 | from rapidfuzz import fuzz |
| 11 | |
| 12 | from tests.utils import RemoteOpenAIServer |
| 13 | |
| 14 | MODEL_NAME = "openai/gpt-oss-20b" |
| 15 | |
| 16 | |
| 17 | @pytest.fixture(scope="module") |
| 18 | def server(): |
| 19 | args = [ |
| 20 | "--max-model-len", |
| 21 | "8192", |
| 22 | "--enforce-eager", |
| 23 | "--enable-auto-tool-choice", |
| 24 | "--tool-call-parser", |
| 25 | "openai", |
| 26 | ] |
| 27 | with RemoteOpenAIServer(MODEL_NAME, args) as remote_server: |
| 28 | yield remote_server |
| 29 | |
| 30 | |
| 31 | @pytest_asyncio.fixture |
| 32 | async def client(server): |
| 33 | """Async fixture providing an OpenAI-compatible vLLM client.""" |
| 34 | async with server.get_async_client() as async_client: |
| 35 | yield async_client |
| 36 | |
| 37 | |
| 38 | # ========================================================== |
| 39 | # Tool Definitions |
| 40 | # ========================================================== |
| 41 | TOOLS = [ |
| 42 | { |
| 43 | "type": "function", |
| 44 | "function": { |
| 45 | "name": "calculator", |
| 46 | "description": "Performs basic arithmetic calculations.", |
| 47 | "parameters": { |
| 48 | "type": "object", |
| 49 | "properties": { |
| 50 | "expression": { |
| 51 | "type": "string", |
| 52 | "description": ( |
| 53 | "Arithmetic expression to evaluate, e.g. '123 + 456'." |
| 54 | ), |
| 55 | } |
| 56 | }, |
| 57 | "required": ["expression"], |
| 58 | }, |
| 59 | }, |
| 60 | }, |
| 61 | { |
| 62 | "type": "function", |
| 63 | "function": { |
| 64 | "name": "get_time", |
| 65 | "description": "Retrieves the current local time for a given city.", |
| 66 | "parameters": { |
| 67 | "type": "object", |
| 68 | "properties": { |
| 69 | "city": { |
| 70 | "type": "string", |
| 71 | "description": "City name, e.g. 'New York'.", |
| 72 | } |
| 73 | }, |
| 74 | "required": ["city"], |
| 75 | }, |
| 76 | }, |
| 77 | }, |
| 78 | ] |
| 79 | |
| 80 | |
| 81 | # ========================================================== |
| 82 | # Message Examples |
| 83 | # ========================================================== |
| 84 | MESSAGES_CALC = [ |
| 85 | {"role": "user", "content": "Calculate 123 + 456 using the calculator."} |
| 86 | ] |
| 87 | |
| 88 | MESSAGES_GET_TIME = [ |
| 89 | {"role": "user", "content": "What is the current time in New York?"} |
| 90 | ] |
| 91 | |
| 92 | MESSAGES_MULTIPLE_CALLS = [ |
| 93 | { |
| 94 | "role": "system", |
| 95 | "content": ( |
| 96 | "You can call multiple tools. " |
| 97 | "When using more than one, return single JSON object with tool_calls array" |
| 98 | "containing each tool call with its function name and arguments. " |
| 99 | "Do not output multiple JSON objects separately." |
| 100 | ), |
| 101 | }, |
| 102 | { |
| 103 | "role": "user", |
| 104 | "content": "First, calculate 7 * 8 using the calculator. " |
| 105 | "Then, use get_time to tell me the current time in New York.", |
| 106 | }, |
| 107 | ] |
| 108 | |
| 109 | MESSAGES_INVALID_CALL = [ |
| 110 | { |
| 111 | "role": "user", |
| 112 | "content": "Can you help with something, " |
| 113 | "but don’t actually perform any calculation?", |
| 114 | } |
| 115 | ] |
| 116 | |
| 117 | |
| 118 | # Expected outputs |
| 119 | FUNC_CALC = "calculator" |
| 120 | FUNC_TIME = "get_time" |
| 121 | |
| 122 | |
| 123 | # ========================================================== |
| 124 | # Utility to extract reasoning and tool calls |
| 125 | # ========================================================== |
| 126 | def extract_reasoning_and_calls(chunks: list) -> tuple[str, list[str], list[str]]: |
| 127 | """Extract accumulated reasoning text and tool call arguments |
| 128 | from streaming chunks. |
| 129 | """ |
| 130 | reasoning: str = "" |
| 131 | tool_calls: dict[int, dict[str, str]] = {} |
| 132 | |
| 133 | for chunk in chunks: |
| 134 | choice = getattr(chunk.choices[0], "delta", None) |
| 135 | if not choice: |
| 136 | continue |
| 137 | |
| 138 | if hasattr(choice, "reasoning") and choice.reasoning: |
| 139 | reasoning += choice.reasoning |
| 140 | |
| 141 | for tc in getattr(choice, "tool_calls", []) or []: |
| 142 | idx = getattr(tc, "index", 0) |
| 143 | tool_entry = tool_calls.setdefault(idx, {"name": "", "arguments": ""}) |
| 144 | |
| 145 | if getattr(tc, "function", None): |
| 146 | func = tc.function |
| 147 | if getattr(func, "name", None): |
| 148 | tool_entry["name"] = func.name |
| 149 | if getattr(func, "arguments", None): |
| 150 | tool_entry["arguments"] += func.arguments |
| 151 | |
| 152 | function_names: list[str] = [v["name"] for _, v in sorted(tool_calls.items())] |
| 153 | arguments: list[str] = [v["arguments"] for _, v in sorted(tool_calls.items())] |
| 154 | |
| 155 | return reasoning, arguments, function_names |
| 156 | |
| 157 | |
| 158 | # ========================================================== |
| 159 | # Test Scenarios |
| 160 | # ========================================================== |
| 161 | @pytest.mark.asyncio |
| 162 | async def test_calculator_tool_call_and_argument_accuracy(client: openai.AsyncOpenAI): |
| 163 | """Verify calculator tool call is made and arguments are accurate.""" |
| 164 | response = await client.chat.completions.create( |
| 165 | model=MODEL_NAME, |
| 166 | messages=MESSAGES_CALC, |
| 167 | tools=TOOLS, |
| 168 | temperature=0.0, |
| 169 | stream=False, |
| 170 | ) |
| 171 | |
| 172 | message = response.choices[0].message |
| 173 | tool_calls = getattr(message, "tool_calls", []) |
| 174 | assert tool_calls, "No tool calls detected" |
| 175 | |
| 176 | calc_call = next((c for c in tool_calls if c.function.name == FUNC_CALC), None) |
| 177 | assert calc_call, "Calculator function not called" |
| 178 | |
| 179 | raw_args = calc_call.function.arguments |
| 180 | assert raw_args, "Calculator arguments missing" |
| 181 | assert "123" in raw_args and "456" in raw_args, ( |
| 182 | f"Expected values not in raw arguments: {raw_args}" |
| 183 | ) |
| 184 | |
| 185 | try: |
| 186 | parsed_args = json.loads(raw_args) |
| 187 | except json.JSONDecodeError: |
| 188 | pytest.fail(f"Invalid JSON in calculator arguments: {raw_args}") |
| 189 | |
| 190 | expected_expr = "123 + 456" |
| 191 | actual_expr = parsed_args.get("expression", "") |
| 192 | similarity = fuzz.ratio(actual_expr, expected_expr) |
| 193 | |
| 194 | assert similarity > 90, ( |
| 195 | f"Expression mismatch: expected '{expected_expr}' " |
| 196 | f"got '{actual_expr}' (similarity={similarity}%)" |
| 197 | ) |
| 198 | |
| 199 | |
| 200 | @pytest.mark.asyncio |
| 201 | async def test_streaming_tool_call_get_time_with_reasoning(client: openai.AsyncOpenAI): |
| 202 | """Verify streamed reasoning and tool call behavior for get_time.""" |
| 203 | stream = await client.chat.completions.create( |
| 204 | model=MODEL_NAME, |
| 205 | messages=MESSAGES_GET_TIME, |
| 206 | tools=TOOLS, |
| 207 | temperature=0.0, |
| 208 | stream=True, |
| 209 | ) |
| 210 | |
| 211 | chunks = [chunk async for chunk in stream] |
| 212 | reasoning, arguments, function_names = extract_reasoning_and_calls(chunks) |
| 213 | |
| 214 | assert FUNC_TIME in function_names, "get_time function not called" |
| 215 | |
| 216 | assert any("New York" in arg for arg in arguments), ( |
| 217 | f"Expected get_time arguments for New York not found in {arguments}" |
| 218 | ) |
| 219 | |
| 220 | assert len(reasoning) > 0, "Expected reasoning content missing" |
| 221 | |
| 222 | assert any(keyword in reasoning for keyword in ["New York", "time", "current"]), ( |
| 223 | f"Reasoning is not relevant to the request: {reasoning}" |
| 224 | ) |
| 225 | |
| 226 | |
| 227 | @pytest.mark.asyncio |
| 228 | async def test_streaming_multiple_tools(client: openai.AsyncOpenAI): |
| 229 | """Test streamed multi-tool response with reasoning.""" |
| 230 | stream = await client.chat.completions.create( |
| 231 | model=MODEL_NAME, |
| 232 | messages=MESSAGES_MULTIPLE_CALLS, |
| 233 | tools=TOOLS, |
| 234 | temperature=0.0, |
| 235 | stream=True, |
| 236 | ) |
| 237 | |
| 238 | chunks = [chunk async for chunk in stream] |
| 239 | reasoning, arguments, function_names = extract_reasoning_and_calls(chunks) |
| 240 | |
| 241 | try: |
| 242 | assert FUNC_CALC in function_names, ( |
| 243 | f"Calculator tool missing — found {function_names}" |
| 244 | ) |
| 245 | assert FUNC_TIME in function_names, ( |
| 246 | f"Time tool missing — found {function_names}" |
| 247 | ) |
| 248 | assert len(reasoning) > 0, "Expected reasoning content in streamed response" |
| 249 | except AssertionError as e: |
| 250 | print(f"ERROR: {e}") |
| 251 | |
| 252 | |
| 253 | @pytest.mark.asyncio |
| 254 | async def test_invalid_tool_call(client: openai.AsyncOpenAI): |
| 255 | """Verify that ambiguous instructions that should not trigger a tool |
| 256 | do not produce any tool calls. |
| 257 | """ |
| 258 | response = await client.chat.completions.create( |
| 259 | model=MODEL_NAME, |
| 260 | messages=MESSAGES_INVALID_CALL, |
| 261 | tools=TOOLS, |
| 262 | temperature=0.0, |
| 263 | stream=False, |
| 264 | ) |
| 265 | |
| 266 | message = response.choices[0].message |
| 267 | |
| 268 | assert message is not None, "Expected message in response" |
| 269 | assert hasattr(message, "content"), "Expected 'content' field in message" |
| 270 | |
| 271 | tool_calls = getattr(message, "tool_calls", []) |
| 272 | assert not tool_calls, ( |
| 273 | f"Model unexpectedly attempted a tool call on invalid input: {tool_calls}" |
| 274 | ) |
| 275 | |
| 276 | |
| 277 | @pytest.mark.asyncio |
| 278 | async def test_tool_call_with_temperature(client: openai.AsyncOpenAI): |
| 279 | """Verify model produces valid tool or text output |
| 280 | under non-deterministic sampling. |
| 281 | """ |
| 282 | response = await client.chat.completions.create( |
| 283 | model=MODEL_NAME, |
| 284 | messages=MESSAGES_CALC, |
| 285 | tools=TOOLS, |
| 286 | temperature=0.7, |
| 287 | stream=False, |
| 288 | ) |
| 289 | |
| 290 | message = response.choices[0].message |
| 291 | assert message is not None, "Expected non-empty message in response" |
| 292 | assert message.tool_calls or message.content, ( |
| 293 | "Response missing both text and tool calls" |
| 294 | ) |
| 295 | |
| 296 | print(f"\nTool calls: {message.tool_calls}") |
| 297 | print(f"Text: {message.content}") |
| 298 | |
| 299 | |
| 300 | @pytest.mark.asyncio |
| 301 | async def test_tool_response_schema_accuracy(client: openai.AsyncOpenAI): |
| 302 | """Validate that tool call arguments adhere to their declared JSON schema.""" |
| 303 | response = await client.chat.completions.create( |
| 304 | model=MODEL_NAME, |
| 305 | messages=MESSAGES_MULTIPLE_CALLS, |
| 306 | tools=TOOLS, |
| 307 | temperature=0.0, |
| 308 | ) |
| 309 | |
| 310 | calls = response.choices[0].message.tool_calls |
| 311 | assert calls, "No tool calls produced" |
| 312 | |
| 313 | for call in calls: |
| 314 | func_name = call.function.name |
| 315 | args = json.loads(call.function.arguments) |
| 316 | |
| 317 | schema: dict[str, object] | None = None |
| 318 | for tool_entry in TOOLS: |
| 319 | function_def = tool_entry.get("function") |
| 320 | if ( |
| 321 | function_def |
| 322 | and isinstance(function_def, dict) |
| 323 | and function_def.get("name") == func_name |
| 324 | ): |
| 325 | schema = function_def.get("parameters") |
| 326 | break |
| 327 | |
| 328 | assert schema is not None, f"No matching tool schema found for {func_name}" |
| 329 | |
| 330 | jsonschema.validate(instance=args, schema=schema) |
| 331 | |
| 332 | |
| 333 | @pytest.mark.asyncio |
| 334 | async def test_semantic_consistency_with_temperature(client: openai.AsyncOpenAI): |
| 335 | """Test that temperature variation doesn't cause contradictory reasoning.""" |
| 336 | responses = [] |
| 337 | for temp in [0.0, 0.5, 1.0]: |
| 338 | resp = await client.chat.completions.create( |
| 339 | model=MODEL_NAME, |
| 340 | messages=MESSAGES_CALC, |
| 341 | tools=TOOLS, |
| 342 | temperature=temp, |
| 343 | ) |
| 344 | text = (resp.choices[0].message.content or "").strip() |
| 345 | responses.append(text) |
| 346 | |
| 347 | # Compare fuzzy similarity between low- and mid-temperature outputs |
| 348 | low_mid_sim = fuzz.ratio(responses[0], responses[1]) |
| 349 | assert low_mid_sim > 60, ( |
| 350 | f"Semantic drift too large between T=0.0 and T=0.5 ({low_mid_sim}%)" |
| 351 | ) |