| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | from collections.abc import Iterable |
| 5 | from typing import Any |
| 6 | |
| 7 | from openai.types.chat import ( |
| 8 | ChatCompletionAssistantMessageParam, |
| 9 | ChatCompletionMessageToolCallParam, |
| 10 | ChatCompletionToolMessageParam, |
| 11 | ) |
| 12 | from openai.types.chat.chat_completion_message_tool_call_param import ( |
| 13 | Function as FunctionCallTool, |
| 14 | ) |
| 15 | from openai.types.responses import ( |
| 16 | ResponseFunctionToolCall, |
| 17 | ResponseOutputItem, |
| 18 | ResponseOutputMessage, |
| 19 | ResponseOutputText, |
| 20 | ResponseReasoningItem, |
| 21 | ) |
| 22 | from openai.types.responses.response import ToolChoice |
| 23 | from openai.types.responses.response_function_tool_call_output_item import ( |
| 24 | ResponseFunctionToolCallOutputItem, |
| 25 | ) |
| 26 | from openai.types.responses.response_output_text import Logprob |
| 27 | from openai.types.responses.response_reasoning_item import ( |
| 28 | Content as ResponseReasoningTextContent, |
| 29 | ) |
| 30 | from openai.types.responses.tool import Tool |
| 31 | |
| 32 | from vllm import envs |
| 33 | from vllm.entrypoints.chat_utils import make_tool_call_id |
| 34 | from vllm.entrypoints.generate.base.protocol import FunctionCall, FunctionDefinition |
| 35 | from vllm.entrypoints.openai.chat_completion.protocol import ( |
| 36 | ChatCompletionMessageParam, |
| 37 | ChatCompletionToolsParam, |
| 38 | ) |
| 39 | from vllm.entrypoints.openai.responses.protocol import ResponseInputOutputItem |
| 40 | from vllm.exceptions import VLLMValidationError |
| 41 | from vllm.logger import init_logger |
| 42 | from vllm.tool_parsers.utils import ( |
| 43 | build_responses_tool_call_name_map, |
| 44 | flat_namespace_tool_name, |
| 45 | iter_response_function_tool_dicts, |
| 46 | resolve_responses_tool_call_name, |
| 47 | ) |
| 48 | from vllm.utils import random_uuid |
| 49 | |
| 50 | logger = init_logger(__name__) |
| 51 | |
| 52 | |
| 53 | def build_response_output_items( |
| 54 | reasoning: str | None, |
| 55 | content: str | None, |
| 56 | tool_calls: list[FunctionCall] | None, |
| 57 | logprobs: list[Logprob] | None = None, |
| 58 | tools: list[Tool] | None = None, |
| 59 | ) -> list[ResponseOutputItem]: |
| 60 | outputs: list[ResponseOutputItem] = [] |
| 61 | tool_call_name_map = build_responses_tool_call_name_map(tools) |
| 62 | |
| 63 | if reasoning: |
| 64 | outputs.append( |
| 65 | ResponseReasoningItem( |
| 66 | id=f"rs_{random_uuid()}", |
| 67 | summary=[], |
| 68 | type="reasoning", |
| 69 | content=[ |
| 70 | ResponseReasoningTextContent(text=reasoning, type="reasoning_text") |
| 71 | ], |
| 72 | status=None, |
| 73 | ) |
| 74 | ) |
| 75 | |
| 76 | if content: |
| 77 | outputs.append( |
| 78 | ResponseOutputMessage( |
| 79 | id=f"msg_{random_uuid()}", |
| 80 | content=[ |
| 81 | ResponseOutputText( |
| 82 | text=content, |
| 83 | annotations=[], |
| 84 | type="output_text", |
| 85 | logprobs=logprobs, |
| 86 | ) |
| 87 | ], |
| 88 | role="assistant", |
| 89 | status="completed", |
| 90 | type="message", |
| 91 | ) |
| 92 | ) |
| 93 | |
| 94 | if tool_calls: |
| 95 | for idx, tool_call in enumerate(tool_calls): |
| 96 | call_name = resolve_responses_tool_call_name( |
| 97 | tool_call.name, tool_call_name_map=tool_call_name_map |
| 98 | ) |
| 99 | outputs.append( |
| 100 | ResponseFunctionToolCall( |
| 101 | id=f"fc_{random_uuid()}", |
| 102 | call_id=tool_call.id |
| 103 | or make_tool_call_id(func_name=tool_call.name, idx=idx), |
| 104 | type="function_call", |
| 105 | status="completed", |
| 106 | name=call_name.name, |
| 107 | namespace=call_name.namespace, |
| 108 | arguments=tool_call.arguments, |
| 109 | ) |
| 110 | ) |
| 111 | |
| 112 | return outputs |
| 113 | |
| 114 | |
| 115 | def should_continue_final_message( |
| 116 | request_input: str | list[ResponseInputOutputItem], |
| 117 | ) -> bool: |
| 118 | """Determine if the last input message is a partial assistant message |
| 119 | that should be continued rather than starting a new generation. |
| 120 | |
| 121 | This enables partial message completion similar to Anthropic's Messages API, |
| 122 | where users can provide an incomplete assistant message and have the model |
| 123 | continue from where it left off. |
| 124 | |
| 125 | A message is considered partial if: |
| 126 | 1. It's a ResponseOutputMessage or ResponseReasoningItem |
| 127 | 2. Its status is "in_progress" or "incomplete" |
| 128 | |
| 129 | Args: |
| 130 | request_input: The input to the Responses API request |
| 131 | |
| 132 | Returns: |
| 133 | True if the final message should be continued, False otherwise |
| 134 | |
| 135 | """ |
| 136 | if isinstance(request_input, str): |
| 137 | # Simple string input is always a user message |
| 138 | return False |
| 139 | |
| 140 | if not request_input: |
| 141 | return False |
| 142 | |
| 143 | last_item = request_input[-1] |
| 144 | |
| 145 | # Check if the last item is a partial assistant message |
| 146 | if isinstance(last_item, ResponseOutputMessage): |
| 147 | return last_item.status in ("in_progress", "incomplete") |
| 148 | |
| 149 | # Check if the last item is a partial reasoning item |
| 150 | if isinstance(last_item, ResponseReasoningItem): |
| 151 | return last_item.status in ("in_progress", "incomplete") |
| 152 | |
| 153 | if isinstance(last_item, dict): |
| 154 | # only support partial completion for messages for now |
| 155 | if last_item.get("type", "message") not in ("message", "reasoning"): |
| 156 | return False |
| 157 | return last_item.get("status") in ("in_progress", "incomplete") |
| 158 | |
| 159 | return False |
| 160 | |
| 161 | |
| 162 | def construct_input_messages( |
| 163 | *, |
| 164 | request_instructions: str | None = None, |
| 165 | request_input: str | list[ResponseInputOutputItem], |
| 166 | prev_msg: list[ChatCompletionMessageParam] | None = None, |
| 167 | prev_response_output: list[ResponseOutputItem] | None = None, |
| 168 | ): |
| 169 | messages: list[ChatCompletionMessageParam] = [] |
| 170 | if request_instructions: |
| 171 | messages.append( |
| 172 | { |
| 173 | "role": "system", |
| 174 | "content": request_instructions, |
| 175 | } |
| 176 | ) |
| 177 | |
| 178 | # Prepend the conversation history. |
| 179 | if prev_msg is not None: |
| 180 | # Filter out system messages from previous conversation -- per the |
| 181 | # OpenAI spec, instructions should NOT carry over across responses. |
| 182 | # The current request's instructions (if any) were already added above. |
| 183 | messages.extend(m for m in prev_msg if m.get("role") != "system") |
| 184 | if prev_response_output is not None: |
| 185 | # Add the previous output. |
| 186 | for output_item in prev_response_output: |
| 187 | # NOTE: We skip the reasoning output. |
| 188 | if isinstance(output_item, ResponseOutputMessage): |
| 189 | for content in output_item.content: |
| 190 | messages.append( |
| 191 | { |
| 192 | "role": "assistant", |
| 193 | "content": content.text, |
| 194 | } |
| 195 | ) |
| 196 | |
| 197 | # Append the new input. |
| 198 | # Responses API supports simple text inputs without chat format. |
| 199 | if isinstance(request_input, str): |
| 200 | messages.append({"role": "user", "content": request_input}) |
| 201 | else: |
| 202 | input_messages = construct_chat_messages_with_tool_call(request_input) |
| 203 | messages.extend(input_messages) |
| 204 | return messages |
| 205 | |
| 206 | |
| 207 | def construct_chat_messages_with_tool_call( |
| 208 | input_messages: list[ResponseInputOutputItem], |
| 209 | ) -> list[ChatCompletionMessageParam]: |
| 210 | """Build chat messages from response items. |
| 211 | |
| 212 | Some chat messages span multiple response items (e.g., reasoning + tool calls). |
| 213 | """ |
| 214 | messages: list[ChatCompletionMessageParam] = [] |
| 215 | for item in input_messages: |
| 216 | message = _construct_message_from_response_item( |
| 217 | item, prev_msg=messages[-1] if messages else None |
| 218 | ) |
| 219 | if message is not None: |
| 220 | messages.append(message) |
| 221 | |
| 222 | return messages |
| 223 | |
| 224 | |
| 225 | def _construct_message_from_response_item( |
| 226 | item: ResponseInputOutputItem, |
| 227 | prev_msg: ChatCompletionMessageParam | None = None, |
| 228 | ) -> ChatCompletionMessageParam | None: |
| 229 | """Returns a new message or None. If `None`, `prev_msg` might be updated. |
| 230 | If `prev_msg` is `None`, a new message is always returned. |
| 231 | """ |
| 232 | prev_assistant_msg = ( |
| 233 | prev_msg if prev_msg and prev_msg.get("role") == "assistant" else None |
| 234 | ) |
| 235 | |
| 236 | if isinstance(item, ResponseFunctionToolCall): |
| 237 | tool_name = item.name |
| 238 | if item.namespace: |
| 239 | tool_name = flat_namespace_tool_name(item.namespace, item.name) |
| 240 | tool_call = ChatCompletionMessageToolCallParam( |
| 241 | id=item.call_id, |
| 242 | function=FunctionCallTool( |
| 243 | name=tool_name, |
| 244 | arguments=item.arguments, |
| 245 | ), |
| 246 | type="function", |
| 247 | ) |
| 248 | if prev_assistant_msg: |
| 249 | tool_calls = prev_assistant_msg.get("tool_calls") |
| 250 | if tool_calls is None: |
| 251 | prev_assistant_msg["tool_calls"] = [tool_call] |
| 252 | return None |
| 253 | if isinstance(tool_calls, list): |
| 254 | tool_calls.append(tool_call) |
| 255 | return None |
| 256 | if isinstance(tool_calls, Iterable) and not isinstance( |
| 257 | tool_calls, (dict, str) |
| 258 | ): |
| 259 | tool_calls = list(tool_calls) |
| 260 | tool_calls.append(tool_call) |
| 261 | prev_assistant_msg["tool_calls"] = tool_calls |
| 262 | return None |
| 263 | logger.warning( |
| 264 | "Previous assistant message has unknown tool_calls format. " |
| 265 | "Tool call merging is skipped and a new assistant message is created. " |
| 266 | "Item %s", |
| 267 | item.id, |
| 268 | ) |
| 269 | return ChatCompletionAssistantMessageParam( |
| 270 | role="assistant", |
| 271 | tool_calls=[tool_call], |
| 272 | ) |
| 273 | elif isinstance(item, ResponseReasoningItem): |
| 274 | reasoning = "" |
| 275 | if item.encrypted_content: |
| 276 | raise VLLMValidationError( |
| 277 | "Encrypted content is not supported.", |
| 278 | parameter="input", |
| 279 | ) |
| 280 | elif item.content and len(item.content) >= 1: |
| 281 | reasoning = item.content[0].text |
| 282 | elif len(item.summary) >= 1: |
| 283 | reasoning = item.summary[0].text |
| 284 | logger.warning( |
| 285 | "Using summary text as reasoning content for item %s. " |
| 286 | "Please use content instead of summary for " |
| 287 | "reasoning items.", |
| 288 | item.id, |
| 289 | ) |
| 290 | |
| 291 | if prev_assistant_msg: |
| 292 | previous_reasoning = prev_assistant_msg.get("reasoning") |
| 293 | if previous_reasoning is None: |
| 294 | prev_assistant_msg["reasoning"] = reasoning |
| 295 | return None |
| 296 | return { |
| 297 | "role": "assistant", |
| 298 | "reasoning": reasoning, |
| 299 | } |
| 300 | elif isinstance(item, ResponseOutputMessage): |
| 301 | output_text = item.content[0].text |
| 302 | if prev_assistant_msg: |
| 303 | previous_content = prev_assistant_msg.get("content") |
| 304 | if previous_content is None: |
| 305 | prev_assistant_msg["content"] = output_text |
| 306 | return None |
| 307 | return { |
| 308 | "role": "assistant", |
| 309 | "content": output_text, |
| 310 | } |
| 311 | elif isinstance(item, ResponseFunctionToolCallOutputItem): |
| 312 | return ChatCompletionToolMessageParam( |
| 313 | role="tool", |
| 314 | content=item.output, |
| 315 | tool_call_id=item.call_id, |
| 316 | ) |
| 317 | elif isinstance(item, dict) and item.get("type") == "function_call_output": |
| 318 | # Append the function call output as a tool message. |
| 319 | return ChatCompletionToolMessageParam( |
| 320 | role="tool", |
| 321 | content=item.get("output"), |
| 322 | tool_call_id=item.get("call_id"), |
| 323 | ) |
| 324 | elif isinstance(item, dict) and item.get("role") == "assistant": |
| 325 | content = item.get("content") |
| 326 | text: str | None = None |
| 327 | if isinstance(content, str): |
| 328 | text = content |
| 329 | elif isinstance(content, list) and content: |
| 330 | text = content[0].get("text") |
| 331 | if text is not None: |
| 332 | if prev_assistant_msg: |
| 333 | previous_content = prev_assistant_msg.get("content") |
| 334 | if previous_content is None: |
| 335 | prev_assistant_msg["content"] = text |
| 336 | return None |
| 337 | return {"role": "assistant", "content": text} |
| 338 | if isinstance(item, dict) and "role" in item: |
| 339 | return item # type: ignore[return-value] |
| 340 | item_type = item.get("type") if isinstance(item, dict) else item.type |
| 341 | raise VLLMValidationError( |
| 342 | f"Unsupported input item type: {item_type}", |
| 343 | parameter="input", |
| 344 | ) |
| 345 | |
| 346 | |
| 347 | def extract_function_tool_names(tools: list[Tool]) -> frozenset[str]: |
| 348 | names = [] |
| 349 | for tool in tools: |
| 350 | if tool.type == "function": |
| 351 | names.append(tool.name) |
| 352 | elif tool.type == "namespace": |
| 353 | names.extend( |
| 354 | flat_namespace_tool_name(tool.name, namespaced_tool.name) |
| 355 | for namespaced_tool in tool.tools |
| 356 | if namespaced_tool.type == "function" |
| 357 | ) |
| 358 | return frozenset(names) |
| 359 | |
| 360 | |
| 361 | def extract_tool_types(tools: list[Tool]) -> set[str]: |
| 362 | """Extracts the tool types from the given tools.""" |
| 363 | tool_types: set[str] = set() |
| 364 | for tool in tools: |
| 365 | if tool.type == "mcp": |
| 366 | # Allow the MCP Tool type to enable built in tools if the |
| 367 | # server_label is allowlisted in |
| 368 | # envs.VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS |
| 369 | if tool.server_label in envs.VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS: |
| 370 | tool_types.add(tool.server_label) |
| 371 | else: |
| 372 | tool_types.add(tool.type) |
| 373 | return tool_types |
| 374 | |
| 375 | |
| 376 | def convert_tool_responses_to_completions_format( |
| 377 | tool: dict, |
| 378 | ) -> ChatCompletionToolsParam: |
| 379 | """Convert a flat Responses tool schema: |
| 380 | {"type": "function", "name": "...", "description": "...", "parameters": {...}} |
| 381 | into a Chat Completions tool param for chat-template rendering. |
| 382 | """ |
| 383 | return ChatCompletionToolsParam( |
| 384 | type="function", |
| 385 | function=FunctionDefinition.model_validate( |
| 386 | {k: v for k, v in tool.items() if k != "type"} |
| 387 | ), |
| 388 | ) |
| 389 | |
| 390 | |
| 391 | def construct_tool_dicts( |
| 392 | tools: list[Tool], |
| 393 | tool_choice: ToolChoice, |
| 394 | exclude_tools_when_tool_choice_none: bool = False, |
| 395 | ) -> list[dict[str, Any]] | None: |
| 396 | if not tools or (tool_choice == "none" and exclude_tools_when_tool_choice_none): |
| 397 | return None |
| 398 | return [ |
| 399 | convert_tool_responses_to_completions_format(tool).model_dump() |
| 400 | for tool in iter_response_function_tool_dicts(tools) |
| 401 | ] |