| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import datetime |
| 5 | from collections.abc import Sequence |
| 6 | from typing import Any |
| 7 | |
| 8 | from openai.types.responses.tool import Tool |
| 9 | from openai_harmony import ( |
| 10 | Author, |
| 11 | Conversation, |
| 12 | DeveloperContent, |
| 13 | HarmonyEncodingName, |
| 14 | Message, |
| 15 | ReasoningEffort, |
| 16 | RenderConversationConfig, |
| 17 | Role, |
| 18 | StreamableParser, |
| 19 | SystemContent, |
| 20 | TextContent, |
| 21 | ToolDescription, |
| 22 | load_harmony_encoding, |
| 23 | ) |
| 24 | |
| 25 | from vllm import envs |
| 26 | from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionToolsParam |
| 27 | from vllm.exceptions import VLLMValidationError |
| 28 | from vllm.logger import init_logger |
| 29 | |
| 30 | logger = init_logger(__name__) |
| 31 | |
| 32 | |
| 33 | def is_function_recipient( |
| 34 | recipient: str, |
| 35 | allowed_function_tool_names: frozenset[str] | None = None, |
| 36 | ) -> bool: |
| 37 | """Check whether *recipient* refers to a function tool call. |
| 38 | |
| 39 | The optional *allowed_function_tool_names* parameter is used by the |
| 40 | Responses API to distinguish bare function-call recipients (missing the |
| 41 | ``functions.`` prefix) from MCP tool calls. When provided, a bare |
| 42 | recipient is only treated as a function call if it appears in the set. |
| 43 | The Chat Completions path omits this parameter so that all bare |
| 44 | recipients are accepted as function calls (the heuristic fallback). |
| 45 | """ |
| 46 | if not recipient: |
| 47 | return False |
| 48 | if recipient.startswith("<|"): |
| 49 | return False |
| 50 | if recipient.startswith("functions."): |
| 51 | return len(recipient) > len("functions.") |
| 52 | if recipient == "assistant": |
| 53 | return False |
| 54 | if recipient in BUILTIN_TOOL_TO_MCP_SERVER_LABEL: |
| 55 | return False |
| 56 | first_segment = recipient.split(".", 1)[0] |
| 57 | if first_segment in BUILTIN_TOOL_TO_MCP_SERVER_LABEL: |
| 58 | return False |
| 59 | if allowed_function_tool_names is not None: |
| 60 | return recipient in allowed_function_tool_names |
| 61 | return True |
| 62 | |
| 63 | |
| 64 | def extract_function_from_recipient(recipient: str) -> str: |
| 65 | return recipient.removeprefix("functions.") |
| 66 | |
| 67 | |
| 68 | REASONING_EFFORT = { |
| 69 | "high": ReasoningEffort.HIGH, |
| 70 | "medium": ReasoningEffort.MEDIUM, |
| 71 | "low": ReasoningEffort.LOW, |
| 72 | } |
| 73 | |
| 74 | _harmony_encoding = None |
| 75 | |
| 76 | # Builtin tools that should be included in the system message when |
| 77 | # they are available and requested by the user. |
| 78 | # Tool args are provided by MCP tool descriptions. Output |
| 79 | # of the tools are stringified. |
| 80 | BUILTIN_TOOL_TO_MCP_SERVER_LABEL: dict[str, str] = { |
| 81 | "python": "code_interpreter", |
| 82 | "browser": "web_search_preview", |
| 83 | "container": "container", |
| 84 | } |
| 85 | |
| 86 | # Derive MCP_BUILTIN_TOOLS from the canonical mapping |
| 87 | MCP_BUILTIN_TOOLS: set[str] = set(BUILTIN_TOOL_TO_MCP_SERVER_LABEL.values()) |
| 88 | |
| 89 | |
| 90 | def has_custom_tools(tool_types: set[str]) -> bool: |
| 91 | """Checks if the given tool types are custom tools |
| 92 | (i.e. any tool other than MCP builtin tools) |
| 93 | """ |
| 94 | return not tool_types.issubset(MCP_BUILTIN_TOOLS) |
| 95 | |
| 96 | |
| 97 | def get_encoding(): |
| 98 | global _harmony_encoding |
| 99 | if _harmony_encoding is None: |
| 100 | _harmony_encoding = load_harmony_encoding(HarmonyEncodingName.HARMONY_GPT_OSS) |
| 101 | return _harmony_encoding |
| 102 | |
| 103 | |
| 104 | def get_system_message( |
| 105 | model_identity: str | None = None, |
| 106 | reasoning_effort: str | None = None, |
| 107 | start_date: str | None = None, |
| 108 | browser_description: str | None = None, |
| 109 | python_description: str | None = None, |
| 110 | container_description: str | None = None, |
| 111 | instructions: str | None = None, |
| 112 | with_custom_tools: bool = False, |
| 113 | ) -> Message: |
| 114 | sys_msg_content = SystemContent.new() |
| 115 | if model_identity is not None: |
| 116 | sys_msg_content = sys_msg_content.with_model_identity(model_identity) |
| 117 | if instructions is not None and envs.VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS: |
| 118 | current_identity = sys_msg_content.model_identity |
| 119 | new_identity = ( |
| 120 | f"{current_identity}\n{instructions}" if current_identity else instructions |
| 121 | ) |
| 122 | sys_msg_content = sys_msg_content.with_model_identity(new_identity) |
| 123 | if reasoning_effort is not None: |
| 124 | if reasoning_effort not in REASONING_EFFORT: |
| 125 | supported_values = ", ".join(REASONING_EFFORT) |
| 126 | raise VLLMValidationError( |
| 127 | f"reasoning_effort={reasoning_effort!r} is not supported by " |
| 128 | f"Harmony. Supported values are: {supported_values}.", |
| 129 | parameter="reasoning_effort", |
| 130 | ) |
| 131 | sys_msg_content = sys_msg_content.with_reasoning_effort( |
| 132 | REASONING_EFFORT[reasoning_effort] |
| 133 | ) |
| 134 | if start_date is None: |
| 135 | # NOTE(woosuk): This brings non-determinism in vLLM. |
| 136 | # Set VLLM_SYSTEM_START_DATE to pin it. |
| 137 | start_date = envs.VLLM_SYSTEM_START_DATE or datetime.datetime.now().strftime( |
| 138 | "%Y-%m-%d" |
| 139 | ) |
| 140 | sys_msg_content = sys_msg_content.with_conversation_start_date(start_date) |
| 141 | if browser_description is not None: |
| 142 | sys_msg_content = sys_msg_content.with_tools(browser_description) |
| 143 | if python_description is not None: |
| 144 | sys_msg_content = sys_msg_content.with_tools(python_description) |
| 145 | if container_description is not None: |
| 146 | sys_msg_content = sys_msg_content.with_tools(container_description) |
| 147 | sys_msg = Message.from_role_and_content(Role.SYSTEM, sys_msg_content) |
| 148 | return sys_msg |
| 149 | |
| 150 | |
| 151 | def create_tool_definition(tool: ChatCompletionToolsParam | Tool): |
| 152 | if isinstance(tool, ChatCompletionToolsParam): |
| 153 | return ToolDescription.new( |
| 154 | name=tool.function.name, |
| 155 | description=tool.function.description or "", |
| 156 | parameters=tool.function.parameters, |
| 157 | ) |
| 158 | return ToolDescription.new( |
| 159 | name=tool.name, |
| 160 | description=tool.description or "", |
| 161 | parameters=tool.parameters, |
| 162 | ) |
| 163 | |
| 164 | |
| 165 | def get_developer_message( |
| 166 | instructions: str | None = None, |
| 167 | tools: list[Tool | ChatCompletionToolsParam] | None = None, |
| 168 | ) -> Message: |
| 169 | dev_msg_content = DeveloperContent.new() |
| 170 | if instructions is not None and not envs.VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS: |
| 171 | dev_msg_content = dev_msg_content.with_instructions(instructions) |
| 172 | if tools is not None: |
| 173 | function_tools: list[Tool | ChatCompletionToolsParam] = [] |
| 174 | for tool in tools: |
| 175 | if tool.type in ( |
| 176 | "web_search_preview", |
| 177 | "code_interpreter", |
| 178 | "container", |
| 179 | ): |
| 180 | pass |
| 181 | |
| 182 | elif tool.type == "function": |
| 183 | function_tools.append(tool) |
| 184 | else: |
| 185 | raise VLLMValidationError( |
| 186 | f"tool type {tool.type!r} is not supported.", |
| 187 | parameter="tools", |
| 188 | ) |
| 189 | if function_tools: |
| 190 | function_tool_descriptions = [ |
| 191 | create_tool_definition(tool) for tool in function_tools |
| 192 | ] |
| 193 | dev_msg_content = dev_msg_content.with_function_tools( |
| 194 | function_tool_descriptions |
| 195 | ) |
| 196 | dev_msg = Message.from_role_and_content(Role.DEVELOPER, dev_msg_content) |
| 197 | return dev_msg |
| 198 | |
| 199 | |
| 200 | def get_user_message(content: str) -> Message: |
| 201 | return Message.from_role_and_content(Role.USER, content) |
| 202 | |
| 203 | |
| 204 | def get_system_or_developer_message(role: str, instructions: str) -> Message: |
| 205 | if role == "system" and envs.VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS: |
| 206 | return get_system_message(instructions=instructions) |
| 207 | return get_developer_message(instructions=instructions) |
| 208 | |
| 209 | |
| 210 | def parse_chat_inputs_to_harmony_messages(chat_msgs: list) -> list[Message]: |
| 211 | """Parse a list of messages from request.messages in the Chat Completion API to |
| 212 | Harmony messages. |
| 213 | """ |
| 214 | msgs: list[Message] = [] |
| 215 | tool_id_names: dict[str, str] = {} |
| 216 | |
| 217 | # Collect tool id to name mappings for tool response recipient values |
| 218 | for chat_msg in chat_msgs: |
| 219 | for tool_call in chat_msg.get("tool_calls", []): |
| 220 | tool_id_names[tool_call.get("id")] = tool_call.get("function", {}).get( |
| 221 | "name" |
| 222 | ) |
| 223 | |
| 224 | for chat_msg in chat_msgs: |
| 225 | msgs.extend(parse_chat_input_to_harmony_message(chat_msg, tool_id_names)) |
| 226 | |
| 227 | return msgs |
| 228 | |
| 229 | |
| 230 | def auto_drop_analysis_messages(msgs: list[Message]) -> list[Message]: |
| 231 | """Harmony models expect the analysis messages (representing raw chain of thought) |
| 232 | to be dropped after an assistant message to the final channel is produced from the |
| 233 | reasoning of those messages. |
| 234 | |
| 235 | The openai-harmony library does this if the very last assistant message is to the |
| 236 | final channel, but it does not handle the case where we're in longer multi-turn |
| 237 | conversations and the client gave us reasoning content from previous turns of |
| 238 | the conversation with multiple assistant messages to the final channel in the |
| 239 | conversation. |
| 240 | |
| 241 | So, we find the index of the last assistant message to the final channel and drop |
| 242 | all analysis messages that precede it, leaving only the analysis messages that |
| 243 | are relevant to the current part of the conversation. |
| 244 | """ |
| 245 | last_assistant_final_index = -1 |
| 246 | for i in range(len(msgs) - 1, -1, -1): |
| 247 | msg = msgs[i] |
| 248 | if msg.author.role == "assistant" and msg.channel == "final": |
| 249 | last_assistant_final_index = i |
| 250 | break |
| 251 | |
| 252 | cleaned_msgs: list[Message] = [] |
| 253 | for i, msg in enumerate(msgs): |
| 254 | if i < last_assistant_final_index and msg.channel == "analysis": |
| 255 | continue |
| 256 | cleaned_msgs.append(msg) |
| 257 | |
| 258 | return cleaned_msgs |
| 259 | |
| 260 | |
| 261 | def flatten_input_text_content(content: Any) -> str | None: |
| 262 | """Extract text parts from a Chat Completion or Responses API content field and |
| 263 | flatten them into a single string. Returns None if no text content is found. |
| 264 | """ |
| 265 | if content is None or isinstance(content, str): |
| 266 | return content |
| 267 | if not isinstance(content, list): |
| 268 | return None |
| 269 | |
| 270 | texts: list[str] = [] |
| 271 | for item in content: |
| 272 | if isinstance(item, str): |
| 273 | texts.append(item) |
| 274 | continue |
| 275 | if isinstance(item, dict): |
| 276 | text = item.get("text") |
| 277 | if text is not None: |
| 278 | texts.append(text) |
| 279 | return "".join(texts) if texts else None |
| 280 | |
| 281 | |
| 282 | def extract_instructions_from_messages( |
| 283 | messages: Sequence[Any], |
| 284 | ) -> tuple[str | None, list[Any]]: |
| 285 | """Peel a leading system/developer Chat Completion or Responses message and |
| 286 | flatten its instruction text. |
| 287 | """ |
| 288 | remaining_messages = list(messages) |
| 289 | if not remaining_messages: |
| 290 | return None, remaining_messages |
| 291 | |
| 292 | first_message = remaining_messages[0] |
| 293 | if not isinstance(first_message, dict): |
| 294 | if hasattr(first_message, "to_dict"): |
| 295 | # Handle OpenAI Harmony Message |
| 296 | first_message = first_message.to_dict() |
| 297 | elif hasattr(first_message, "model_dump"): |
| 298 | first_message = first_message.model_dump(exclude_none=True) |
| 299 | else: |
| 300 | raise VLLMValidationError( |
| 301 | f"Unknown message type: {type(first_message)}", |
| 302 | parameter="input", |
| 303 | ) |
| 304 | |
| 305 | if first_message.get("role") not in ( |
| 306 | "system", |
| 307 | "developer", |
| 308 | ): |
| 309 | return None, remaining_messages |
| 310 | |
| 311 | instructions = flatten_input_text_content(first_message.get("content")) |
| 312 | return instructions, remaining_messages[1:] |
| 313 | |
| 314 | |
| 315 | def build_harmony_preamble( |
| 316 | *, |
| 317 | instructions: str | None = None, |
| 318 | tools: list[Tool | ChatCompletionToolsParam] | None = None, |
| 319 | reasoning_effort: str | None = None, |
| 320 | browser_description: str | None = None, |
| 321 | python_description: str | None = None, |
| 322 | container_description: str | None = None, |
| 323 | with_custom_tools: bool = False, |
| 324 | ) -> list[Message]: |
| 325 | """Build the standard Harmony system/developer prefix for a request.""" |
| 326 | developer_instructions = system_instructions = None |
| 327 | if envs.VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS: |
| 328 | system_instructions = instructions |
| 329 | else: |
| 330 | developer_instructions = instructions |
| 331 | |
| 332 | messages = [ |
| 333 | get_system_message( |
| 334 | reasoning_effort=reasoning_effort, |
| 335 | browser_description=browser_description, |
| 336 | python_description=python_description, |
| 337 | container_description=container_description, |
| 338 | instructions=system_instructions, |
| 339 | with_custom_tools=with_custom_tools, |
| 340 | ) |
| 341 | ] |
| 342 | if developer_instructions or tools: |
| 343 | messages.append( |
| 344 | get_developer_message( |
| 345 | instructions=developer_instructions, |
| 346 | tools=tools, |
| 347 | ) |
| 348 | ) |
| 349 | return messages |
| 350 | |
| 351 | |
| 352 | def parse_chat_input_to_harmony_message( |
| 353 | chat_msg, tool_id_names: dict[str, str] | None = None |
| 354 | ) -> list[Message]: |
| 355 | """Parse a message from request.messages in the Chat Completion API to |
| 356 | Harmony messages. |
| 357 | """ |
| 358 | tool_id_names = tool_id_names or {} |
| 359 | |
| 360 | if not isinstance(chat_msg, dict): |
| 361 | # Handle Pydantic models |
| 362 | chat_msg = chat_msg.model_dump(exclude_none=True) |
| 363 | |
| 364 | role = chat_msg.get("role") |
| 365 | msgs: list[Message] = [] |
| 366 | |
| 367 | # Assistant message with tool calls |
| 368 | tool_calls = chat_msg.get("tool_calls", []) |
| 369 | |
| 370 | if role == "assistant" and tool_calls: |
| 371 | content = flatten_input_text_content(chat_msg.get("content")) |
| 372 | if content: |
| 373 | commentary_msg = Message.from_role_and_content(Role.ASSISTANT, content) |
| 374 | commentary_msg = commentary_msg.with_channel("commentary") |
| 375 | msgs.append(commentary_msg) |
| 376 | |
| 377 | reasoning = chat_msg.get("reasoning") |
| 378 | if reasoning: |
| 379 | analysis_msg = Message.from_role_and_content(Role.ASSISTANT, reasoning) |
| 380 | analysis_msg = analysis_msg.with_channel("analysis") |
| 381 | msgs.append(analysis_msg) |
| 382 | |
| 383 | for call in tool_calls: |
| 384 | func = call.get("function", {}) |
| 385 | name = func.get("name", "") |
| 386 | arguments = func.get("arguments", "") or "" |
| 387 | msg = Message.from_role_and_content(Role.ASSISTANT, arguments) |
| 388 | msg = msg.with_channel("commentary") |
| 389 | msg = msg.with_recipient(f"functions.{name}") |
| 390 | # Officially, this should be `<|constrain|>json` but there is not clear |
| 391 | # evidence that improves accuracy over `json` and some anecdotes to the |
| 392 | # contrary. Further testing of the different content_types is needed. |
| 393 | msg = msg.with_content_type("json") |
| 394 | msgs.append(msg) |
| 395 | return msgs |
| 396 | |
| 397 | # Tool role message (tool output) |
| 398 | if role == "tool": |
| 399 | tool_call_id = chat_msg.get("tool_call_id", "") |
| 400 | name = tool_id_names.get(tool_call_id, "") |
| 401 | content = flatten_input_text_content(chat_msg.get("content")) or "" |
| 402 | |
| 403 | msg = ( |
| 404 | Message.from_author_and_content( |
| 405 | Author.new(Role.TOOL, f"functions.{name}"), content |
| 406 | ) |
| 407 | .with_channel("commentary") |
| 408 | .with_recipient("assistant") |
| 409 | ) |
| 410 | return [msg] |
| 411 | |
| 412 | # Non-tool reasoning content |
| 413 | reasoning = chat_msg.get("reasoning") |
| 414 | if role == "assistant" and reasoning: |
| 415 | analysis_msg = Message.from_role_and_content(Role.ASSISTANT, reasoning) |
| 416 | analysis_msg = analysis_msg.with_channel("analysis") |
| 417 | msgs.append(analysis_msg) |
| 418 | |
| 419 | # Default: user/assistant/system messages with content |
| 420 | content = chat_msg.get("content") or "" |
| 421 | if content is None: |
| 422 | content = "" |
| 423 | if isinstance(content, str): |
| 424 | contents = [TextContent(text=content)] |
| 425 | else: |
| 426 | # TODO: Support refusal. |
| 427 | contents = [TextContent(text=c.get("text", "")) for c in content] |
| 428 | |
| 429 | # Only add assistant messages if they have content, as reasoning or tool calling |
| 430 | # assistant messages were already added above. |
| 431 | if role == "assistant" and contents and contents[0].text: |
| 432 | msg = Message.from_role_and_contents(role, contents) |
| 433 | # Send non-tool assistant messages to the final channel |
| 434 | msg = msg.with_channel("final") |
| 435 | msgs.append(msg) |
| 436 | elif role in ("system", "developer"): |
| 437 | instructions = flatten_input_text_content(chat_msg.get("content")) |
| 438 | if instructions is not None: |
| 439 | msg = get_system_or_developer_message(role, instructions) |
| 440 | msgs.append(msg) |
| 441 | # For user messages, add them directly even if no content. |
| 442 | elif role != "assistant": |
| 443 | msg = Message.from_role_and_contents(role, contents) |
| 444 | msgs.append(msg) |
| 445 | |
| 446 | return msgs |
| 447 | |
| 448 | |
| 449 | def render_for_completion(messages: list[Message]) -> list[int]: |
| 450 | messages = auto_drop_analysis_messages(messages) |
| 451 | conversation = Conversation.from_messages(messages) |
| 452 | encoding = get_encoding() |
| 453 | token_ids = encoding.render_conversation_for_completion( |
| 454 | conversation, |
| 455 | Role.ASSISTANT, |
| 456 | config=RenderConversationConfig(auto_drop_analysis=False), |
| 457 | ) |
| 458 | return _use_legacy_tool_call_headers(token_ids) |
| 459 | |
| 460 | |
| 461 | def _use_legacy_tool_call_headers(token_ids: list[int]) -> list[int]: |
| 462 | """Keep prior assistant tool calls in the format expected by GPT-OSS. |
| 463 | |
| 464 | oss-harmony 0.0.10 changed a rendered tool-call header from:: |
| 465 | |
| 466 | assistant to=python<|channel|>commentary code |
| 467 | |
| 468 | to:: |
| 469 | |
| 470 | assistant<|channel|>commentary to=python <|constrain|>code |
| 471 | |
| 472 | GPT-OSS can echo the latter as a malformed header with the recipient in |
| 473 | both positions. Rewrite only that unambiguous token-level header shape, |
| 474 | while retaining oss-harmony's embedded vocabulary and parser. |
| 475 | """ |
| 476 | encoding = get_encoding() |
| 477 | start = encoding.encode("<|start|>", allowed_special="all")[0] |
| 478 | channel_token = encoding.encode("<|channel|>", allowed_special="all")[0] |
| 479 | constrain = encoding.encode("<|constrain|>", allowed_special="all")[0] |
| 480 | message = encoding.encode("<|message|>", allowed_special="all")[0] |
| 481 | |
| 482 | result = token_ids.copy() |
| 483 | index = 0 |
| 484 | while index < len(result): |
| 485 | if result[index] != start: |
| 486 | index += 1 |
| 487 | continue |
| 488 | |
| 489 | try: |
| 490 | header_end = result.index(message, index + 1) |
| 491 | channel_index = result.index(channel_token, index + 1, header_end) |
| 492 | constrain_index = result.index(constrain, channel_index + 1, header_end) |
| 493 | except ValueError: |
| 494 | index += 1 |
| 495 | continue |
| 496 | |
| 497 | author = encoding.decode(result[index + 1 : channel_index]) |
| 498 | channel_and_recipient = encoding.decode( |
| 499 | result[channel_index + 1 : constrain_index] |
| 500 | ) |
| 501 | content_type = encoding.decode(result[constrain_index + 1 : header_end]) |
| 502 | if ( |
| 503 | author != "assistant" |
| 504 | or " to=" not in channel_and_recipient |
| 505 | or not channel_and_recipient.endswith(" ") |
| 506 | ): |
| 507 | index = header_end + 1 |
| 508 | continue |
| 509 | |
| 510 | channel, recipient = channel_and_recipient[:-1].split(" to=", 1) |
| 511 | if not channel or not recipient or not content_type: |
| 512 | index = header_end + 1 |
| 513 | continue |
| 514 | |
| 515 | legacy_header = ( |
| 516 | [start] |
| 517 | + encoding.encode(f"assistant to={recipient}") |
| 518 | + [channel_token] |
| 519 | + encoding.encode(f"{channel} {content_type}") |
| 520 | + [message] |
| 521 | ) |
| 522 | result[index : header_end + 1] = legacy_header |
| 523 | index += len(legacy_header) |
| 524 | |
| 525 | return result |
| 526 | |
| 527 | |
| 528 | def get_streamable_parser_for_assistant() -> StreamableParser: |
| 529 | return StreamableParser(get_encoding(), role=Role.ASSISTANT) |