| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | import time |
| 4 | from collections.abc import Sequence |
| 5 | from dataclasses import dataclass |
| 6 | from http import HTTPStatus |
| 7 | from typing import TYPE_CHECKING, Any, TypeAlias, cast |
| 8 | |
| 9 | from openai.types.responses import ResponseFunctionToolCall, ResponseOutputItem |
| 10 | from openai.types.responses.tool import Mcp, Tool |
| 11 | from openai_harmony import Message as OpenAIMessage |
| 12 | from openai_harmony import ToolNamespaceConfig |
| 13 | |
| 14 | from vllm.config import ModelConfig |
| 15 | from vllm.entrypoints.chat_utils import ( |
| 16 | ChatCompletionMessageParam, |
| 17 | ChatTemplateContentFormatOption, |
| 18 | ConversationMessage, |
| 19 | has_non_text_content, |
| 20 | ) |
| 21 | from vllm.entrypoints.generate.base.protocol import validate_request_mm_kwargs |
| 22 | from vllm.entrypoints.openai.chat_completion.protocol import ( |
| 23 | ChatCompletionNamedToolChoiceParam, |
| 24 | ChatCompletionRequest, |
| 25 | ) |
| 26 | from vllm.entrypoints.openai.completion.protocol import ( |
| 27 | CompletionRequest, |
| 28 | ) |
| 29 | from vllm.entrypoints.openai.parser.harmony_utils import ( |
| 30 | BUILTIN_TOOL_TO_MCP_SERVER_LABEL, |
| 31 | build_harmony_preamble, |
| 32 | extract_instructions_from_messages, |
| 33 | get_user_message, |
| 34 | has_custom_tools, |
| 35 | parse_chat_inputs_to_harmony_messages, |
| 36 | render_for_completion, |
| 37 | ) |
| 38 | from vllm.entrypoints.openai.responses.harmony import ( |
| 39 | construct_harmony_previous_input_messages, |
| 40 | response_input_to_harmony, |
| 41 | ) |
| 42 | from vllm.entrypoints.openai.responses.protocol import ResponsesRequest |
| 43 | from vllm.entrypoints.openai.responses.utils import ( |
| 44 | construct_input_messages, |
| 45 | construct_tool_dicts, |
| 46 | extract_tool_types, |
| 47 | ) |
| 48 | from vllm.entrypoints.serve import create_error_response |
| 49 | from vllm.entrypoints.serve.engine.protocol import ErrorResponse |
| 50 | from vllm.entrypoints.serve.utils.request_logger import RequestLogger |
| 51 | from vllm.exceptions import VLLMValidationError |
| 52 | from vllm.inputs import ( |
| 53 | EngineInput, |
| 54 | PromptType, |
| 55 | SingletonPrompt, |
| 56 | TokensPrompt, |
| 57 | tokens_input, |
| 58 | ) |
| 59 | from vllm.logger import init_logger |
| 60 | from vllm.parser import Parser, ParserManager |
| 61 | from vllm.renderers import BaseRenderer, ChatParams, TokenizeParams, merge_kwargs |
| 62 | from vllm.renderers.inputs.preprocess import ( |
| 63 | parse_model_prompt, |
| 64 | prompt_to_seq, |
| 65 | ) |
| 66 | from vllm.utils.mistral import is_mistral_tokenizer, is_mistral_tool_parser |
| 67 | from vllm.utils.mistral import mt as _mt |
| 68 | |
| 69 | logger = init_logger(__name__) |
| 70 | |
| 71 | if TYPE_CHECKING: |
| 72 | from vllm.entrypoints.mcp.tool_server import ToolServer |
| 73 | |
| 74 | |
| 75 | ResponsesPreviousMessages: TypeAlias = ( |
| 76 | list[ChatCompletionMessageParam] | list[OpenAIMessage] |
| 77 | ) |
| 78 | |
| 79 | |
| 80 | @dataclass(frozen=True) |
| 81 | class ResponsesRenderResult: |
| 82 | messages: ResponsesPreviousMessages |
| 83 | engine_input: EngineInput |
| 84 | |
| 85 | |
| 86 | def _extract_allowed_tools_from_mcp_requests( |
| 87 | tools: list[Tool], |
| 88 | ) -> dict[str, list[str] | None]: |
| 89 | allowed_tools_map: dict[str, list[str] | None] = {} |
| 90 | for tool in tools: |
| 91 | if not isinstance(tool, Mcp): |
| 92 | continue |
| 93 | |
| 94 | allowed_tools = None |
| 95 | if isinstance(tool.allowed_tools, list): |
| 96 | allowed_tools = tool.allowed_tools |
| 97 | elif tool.allowed_tools is not None: |
| 98 | allowed_tools = tool.allowed_tools.tool_names |
| 99 | if allowed_tools is not None and "*" in allowed_tools: |
| 100 | allowed_tools = None |
| 101 | allowed_tools_map[tool.server_label] = allowed_tools |
| 102 | return allowed_tools_map |
| 103 | |
| 104 | |
| 105 | def _reused_prompt_token_ids( |
| 106 | request: Any, renderer: BaseRenderer, messages: list[Any] | None = None |
| 107 | ) -> list[int] | None: |
| 108 | """Pop prompt token ids forwarded for decode-side reuse, if any. |
| 109 | |
| 110 | Disaggregated serving carries the prefill stage's ids in |
| 111 | ``kv_transfer_params`` so the decode stage can skip re-tokenizing. Removing |
| 112 | the key keeps the id list out of the engine's sampling metadata. |
| 113 | |
| 114 | Returns None without checking the ids when ``echo`` is set or ``messages`` |
| 115 | has non-text content, since both need ``messages`` to be rendered. |
| 116 | Otherwise raises VLLMValidationError if the ids are malformed or out of |
| 117 | vocabulary. |
| 118 | """ |
| 119 | kv = getattr(request, "kv_transfer_params", None) |
| 120 | if not isinstance(kv, dict): |
| 121 | return None |
| 122 | ids = kv.pop("prompt_token_ids", None) |
| 123 | if ids is None: |
| 124 | return None |
| 125 | if getattr(request, "echo", False): |
| 126 | logger.debug( |
| 127 | "Ignoring kv_transfer_params['prompt_token_ids']: " |
| 128 | "echo is set, so messages are rendered instead." |
| 129 | ) |
| 130 | return None |
| 131 | if has_non_text_content(messages): |
| 132 | logger.debug( |
| 133 | "Ignoring kv_transfer_params['prompt_token_ids']: " |
| 134 | "messages have non-text content and are rendered instead." |
| 135 | ) |
| 136 | return None |
| 137 | # bool is an int subclass, hence the exact type check. |
| 138 | if ( |
| 139 | not isinstance(ids, list) |
| 140 | or not ids |
| 141 | or any(type(x) is not int or x < 0 for x in ids) |
| 142 | ): |
| 143 | raise VLLMValidationError( |
| 144 | "`kv_transfer_params['prompt_token_ids']` must be a non-empty list " |
| 145 | "of non-negative integers.", |
| 146 | parameter="kv_transfer_params.prompt_token_ids", |
| 147 | ) |
| 148 | # The engine checks this too, but only after a streamed response starts. |
| 149 | renderer.validate_token_ids(ids, parameter="kv_transfer_params.prompt_token_ids") |
| 150 | return ids |
| 151 | |
| 152 | |
| 153 | class OnlineRenderer: |
| 154 | def __init__( |
| 155 | self, |
| 156 | model_config: ModelConfig, |
| 157 | renderer: BaseRenderer, |
| 158 | *, |
| 159 | request_logger: RequestLogger | None, |
| 160 | chat_template: str | None, |
| 161 | chat_template_content_format: ChatTemplateContentFormatOption, |
| 162 | trust_request_chat_template: bool = False, |
| 163 | trust_request_mm_kwargs: bool = False, |
| 164 | enable_auto_tools: bool = False, |
| 165 | exclude_tools_when_tool_choice_none: bool = False, |
| 166 | tool_parser: str | None = None, |
| 167 | reasoning_parser: str | None = None, |
| 168 | tool_strict_level: str = "auto", |
| 169 | default_chat_template_kwargs: dict[str, Any] | None = None, |
| 170 | log_error_stack: bool = False, |
| 171 | ) -> None: |
| 172 | self.model_config = model_config |
| 173 | self.renderer = renderer |
| 174 | self.request_logger = request_logger |
| 175 | |
| 176 | self.enable_auto_tools = enable_auto_tools |
| 177 | self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none |
| 178 | self.use_harmony = model_config.hf_config.model_type == "gpt_oss" |
| 179 | self.parser: type[Parser] | None = ParserManager.get_parser( |
| 180 | tool_parser_name=tool_parser, |
| 181 | reasoning_parser_name=reasoning_parser, |
| 182 | enable_auto_tools=enable_auto_tools, |
| 183 | tool_strict_level=tool_strict_level, |
| 184 | model_name=model_config.model, |
| 185 | is_harmony=self.use_harmony, |
| 186 | tokenizer=renderer.tokenizer, |
| 187 | ) |
| 188 | |
| 189 | self.chat_template = chat_template |
| 190 | self.chat_template_content_format: ChatTemplateContentFormatOption = ( |
| 191 | chat_template_content_format |
| 192 | ) |
| 193 | self.default_chat_template_kwargs: dict[str, Any] = ( |
| 194 | default_chat_template_kwargs or {} |
| 195 | ) |
| 196 | self.trust_request_chat_template = trust_request_chat_template |
| 197 | self.trust_request_mm_kwargs = trust_request_mm_kwargs |
| 198 | |
| 199 | self.log_error_stack = log_error_stack |
| 200 | self.supports_browsing = False |
| 201 | self.supports_code_interpreter = False |
| 202 | |
| 203 | def warmup(self) -> None: |
| 204 | self.renderer.warmup( |
| 205 | ChatParams( |
| 206 | chat_template=self.chat_template, |
| 207 | chat_template_content_format=self.chat_template_content_format, |
| 208 | chat_template_kwargs=self.default_chat_template_kwargs, |
| 209 | ) |
| 210 | ) |
| 211 | |
| 212 | async def render_chat( |
| 213 | self, |
| 214 | request: ChatCompletionRequest, |
| 215 | *, |
| 216 | skip_mm_cache: bool = False, |
| 217 | ) -> tuple[list[ConversationMessage], list[EngineInput]] | ErrorResponse: |
| 218 | """Core preprocessing logic for chat requests (no model/engine check). |
| 219 | |
| 220 | Called directly by render_chat_request and delegated to by |
| 221 | OpenAIServingChat.render_chat_request after its engine-aware checks. |
| 222 | |
| 223 | Decode-side token reuse (ids forwarded in ``kv_transfer_params``) is |
| 224 | handled deeper, in ``preprocess_chat`` / ``_make_request_with_harmony``, |
| 225 | so it skips only templating and tokenization while tool-choice |
| 226 | validation and ``adjust_request`` still run and the output is |
| 227 | detokenized (text-out). |
| 228 | """ |
| 229 | tokenizer = self.renderer.tokenizer |
| 230 | |
| 231 | tool_parser = self.parser.tool_parser_cls if self.parser is not None else None |
| 232 | |
| 233 | if is_mistral_tokenizer(tokenizer): |
| 234 | # because of issues with pydantic we need to potentially |
| 235 | # re-serialize the tool_calls field of the request |
| 236 | _mt.maybe_serialize_tool_calls(request) # type: ignore[arg-type] |
| 237 | _mt.truncate_tool_call_ids(request) # type: ignore[arg-type] |
| 238 | _mt.validate_request_params(request) |
| 239 | |
| 240 | # Check if tool parsing is unavailable (common condition) |
| 241 | tool_parsing_unavailable = ( |
| 242 | tool_parser is None |
| 243 | and not is_mistral_tokenizer(tokenizer) |
| 244 | and not self.use_harmony |
| 245 | ) |
| 246 | |
| 247 | # Validate tool_choice when tool parsing is required but unavailable |
| 248 | if tool_parsing_unavailable and request.tool_choice not in ( |
| 249 | None, |
| 250 | "none", |
| 251 | ): |
| 252 | if request.tool_choice == "auto" and not self.enable_auto_tools: |
| 253 | # for hf tokenizers, "auto" tools requires |
| 254 | # --enable-auto-tool-choice and --tool-call-parser |
| 255 | return self.create_error_response( |
| 256 | '"auto" tool choice requires ' |
| 257 | "--enable-auto-tool-choice and --tool-call-parser to be set" |
| 258 | ) |
| 259 | elif request.tool_choice != "auto": |
| 260 | # "required" or named tool requires tool parser |
| 261 | if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam): |
| 262 | tool_choice_desc = f'function "{request.tool_choice.function.name}"' |
| 263 | else: |
| 264 | tool_choice_desc = f'"{request.tool_choice}"' |
| 265 | return self.create_error_response( |
| 266 | f"tool_choice={tool_choice_desc} requires " |
| 267 | "--tool-call-parser to be set" |
| 268 | ) |
| 269 | |
| 270 | if request.tools is None or ( |
| 271 | request.tool_choice == "none" and self.exclude_tools_when_tool_choice_none |
| 272 | ): |
| 273 | tool_dicts = None |
| 274 | else: |
| 275 | tool_dicts = [tool.model_dump() for tool in request.tools] |
| 276 | |
| 277 | if not self.use_harmony: |
| 278 | # Common case. |
| 279 | error_check_ret = self.validate_chat_template( |
| 280 | request_chat_template=request.chat_template, |
| 281 | chat_template_kwargs=request.chat_template_kwargs, |
| 282 | trust_request_chat_template=self.trust_request_chat_template, |
| 283 | ) |
| 284 | if error_check_ret is not None: |
| 285 | return error_check_ret |
| 286 | |
| 287 | conversation, engine_inputs = await self.preprocess_chat( |
| 288 | request, |
| 289 | request.messages, |
| 290 | default_template=self.chat_template, |
| 291 | default_template_content_format=self.chat_template_content_format, |
| 292 | default_template_kwargs=self.default_chat_template_kwargs, |
| 293 | tool_dicts=tool_dicts, |
| 294 | parser=self.parser, |
| 295 | skip_mm_cache=skip_mm_cache, |
| 296 | ) |
| 297 | else: |
| 298 | # For GPT-OSS. |
| 299 | self.adjust_harmony_request(request) |
| 300 | should_include_tools = tool_dicts is not None |
| 301 | conversation, engine_inputs = self._make_request_with_harmony( |
| 302 | request, should_include_tools |
| 303 | ) |
| 304 | |
| 305 | return conversation, engine_inputs |
| 306 | |
| 307 | async def render_responses( |
| 308 | self, |
| 309 | request: ResponsesRequest, |
| 310 | *, |
| 311 | previous_messages: ResponsesPreviousMessages | None = None, |
| 312 | previous_response_outputs: list[ResponseOutputItem] | None = None, |
| 313 | tool_server: "ToolServer | None" = None, |
| 314 | skip_mm_cache: bool = False, |
| 315 | ) -> ResponsesRenderResult | ErrorResponse: |
| 316 | """Render a Responses request using only explicitly supplied history.""" |
| 317 | template_error = self.validate_chat_template( |
| 318 | request_chat_template=None, |
| 319 | chat_template_kwargs=request.chat_template_kwargs, |
| 320 | trust_request_chat_template=self.trust_request_chat_template, |
| 321 | ) |
| 322 | if template_error is not None: |
| 323 | return template_error |
| 324 | |
| 325 | if self.use_harmony: |
| 326 | return self._render_responses_with_harmony( |
| 327 | request, |
| 328 | previous_messages=previous_messages, |
| 329 | previous_response_outputs=previous_response_outputs, |
| 330 | tool_server=tool_server, |
| 331 | ) |
| 332 | |
| 333 | if previous_messages is not None and any( |
| 334 | isinstance(message, OpenAIMessage) for message in previous_messages |
| 335 | ): |
| 336 | return self.create_error_response( |
| 337 | "Non-Harmony Responses history must use chat messages.", |
| 338 | err_type="invalid_request_error", |
| 339 | param="previous_response_id", |
| 340 | ) |
| 341 | |
| 342 | tool_dicts = construct_tool_dicts( |
| 343 | request.tools, |
| 344 | request.tool_choice, |
| 345 | exclude_tools_when_tool_choice_none=( |
| 346 | self.exclude_tools_when_tool_choice_none |
| 347 | ), |
| 348 | ) |
| 349 | messages = construct_input_messages( |
| 350 | request_instructions=request.instructions, |
| 351 | request_input=request.input, |
| 352 | prev_msg=list(previous_messages) if previous_messages is not None else None, |
| 353 | prev_response_output=( |
| 354 | list(previous_response_outputs) |
| 355 | if previous_response_outputs is not None |
| 356 | else None |
| 357 | ), |
| 358 | ) |
| 359 | chat_template_kwargs = ( |
| 360 | request.build_chat_params( |
| 361 | self.chat_template, |
| 362 | self.chat_template_content_format, |
| 363 | ) |
| 364 | .with_defaults(self.default_chat_template_kwargs) |
| 365 | .chat_template_kwargs |
| 366 | ) |
| 367 | _, engine_inputs = await self.preprocess_chat( |
| 368 | request, |
| 369 | messages, |
| 370 | default_template=self.chat_template, |
| 371 | default_template_content_format=self.chat_template_content_format, |
| 372 | default_template_kwargs=chat_template_kwargs, |
| 373 | tool_dicts=tool_dicts, |
| 374 | parser=self.parser, |
| 375 | skip_mm_cache=skip_mm_cache, |
| 376 | ) |
| 377 | return self._responses_render_result(messages, engine_inputs) |
| 378 | |
| 379 | def _render_responses_with_harmony( |
| 380 | self, |
| 381 | request: ResponsesRequest, |
| 382 | *, |
| 383 | previous_messages: ResponsesPreviousMessages | None, |
| 384 | previous_response_outputs: list[ResponseOutputItem] | None, |
| 385 | tool_server: "ToolServer | None", |
| 386 | ) -> ResponsesRenderResult | ErrorResponse: |
| 387 | self.adjust_harmony_request(request) |
| 388 | |
| 389 | if previous_messages is not None and any( |
| 390 | not isinstance(message, OpenAIMessage) for message in previous_messages |
| 391 | ): |
| 392 | return self.create_error_response( |
| 393 | "Harmony Responses history must use Harmony messages.", |
| 394 | err_type="invalid_request_error", |
| 395 | param="previous_response_id", |
| 396 | ) |
| 397 | |
| 398 | messages: list[OpenAIMessage] = [] |
| 399 | request_input = request.input |
| 400 | if previous_messages is None: |
| 401 | tool_types = extract_tool_types(request.tools) |
| 402 | with_custom_tools = has_custom_tools(tool_types) |
| 403 | instructions = request.instructions |
| 404 | if instructions is None and isinstance(request_input, list): |
| 405 | instructions, request_input = extract_instructions_from_messages( |
| 406 | request_input |
| 407 | ) |
| 408 | descriptions = self._get_harmony_builtin_tool_descriptions( |
| 409 | request.tools, |
| 410 | tool_types, |
| 411 | tool_server, |
| 412 | ) |
| 413 | messages.extend( |
| 414 | build_harmony_preamble( |
| 415 | instructions=instructions, |
| 416 | tools=request.tools if with_custom_tools else None, |
| 417 | reasoning_effort=( |
| 418 | request.reasoning.effort if request.reasoning else None |
| 419 | ), |
| 420 | with_custom_tools=with_custom_tools, |
| 421 | **descriptions, |
| 422 | ) |
| 423 | ) |
| 424 | messages.extend(construct_harmony_previous_input_messages(request)) |
| 425 | else: |
| 426 | messages.extend(previous_messages) |
| 427 | |
| 428 | previous_outputs = list(previous_response_outputs or ()) |
| 429 | try: |
| 430 | if isinstance(request_input, str): |
| 431 | if request_input or not request.previous_input_messages: |
| 432 | messages.append(get_user_message(request_input)) |
| 433 | else: |
| 434 | for response_message in request_input: |
| 435 | new_message = response_input_to_harmony( |
| 436 | response_message, |
| 437 | previous_outputs, |
| 438 | ) |
| 439 | if new_message is not None: |
| 440 | messages.append(new_message) |
| 441 | if isinstance(response_message, ResponseFunctionToolCall): |
| 442 | previous_outputs.append(response_message) |
| 443 | except (ValueError, VLLMValidationError) as exc: |
| 444 | return self.create_error_response( |
| 445 | str(exc), |
| 446 | err_type="invalid_request_error", |
| 447 | param="input", |
| 448 | ) |
| 449 | |
| 450 | return ResponsesRenderResult( |
| 451 | messages=messages, |
| 452 | engine_input=self.render_responses_harmony_messages( |
| 453 | messages, |
| 454 | cache_salt=request.cache_salt, |
| 455 | tok_params=request.build_tok_params(self.model_config), |
| 456 | ), |
| 457 | ) |
| 458 | |
| 459 | def _get_harmony_builtin_tool_descriptions( |
| 460 | self, |
| 461 | tools: list[Tool], |
| 462 | tool_types: set[str], |
| 463 | tool_server: "ToolServer | None", |
| 464 | ) -> dict[str, ToolNamespaceConfig | None]: |
| 465 | allowed_tools = _extract_allowed_tools_from_mcp_requests(tools) |
| 466 | descriptions: dict[str, ToolNamespaceConfig | None] = {} |
| 467 | for server_name, request_name in BUILTIN_TOOL_TO_MCP_SERVER_LABEL.items(): |
| 468 | description = ( |
| 469 | tool_server.get_tool_description( |
| 470 | server_name, |
| 471 | allowed_tools.get(request_name), |
| 472 | ) |
| 473 | if request_name in tool_types |
| 474 | and tool_server is not None |
| 475 | and tool_server.has_tool(server_name) |
| 476 | else None |
| 477 | ) |
| 478 | descriptions[f"{server_name}_description"] = description |
| 479 | return descriptions |
| 480 | |
| 481 | def render_responses_harmony_messages( |
| 482 | self, |
| 483 | messages: list[OpenAIMessage], |
| 484 | *, |
| 485 | cache_salt: str | None, |
| 486 | tok_params: TokenizeParams | None = None, |
| 487 | ) -> EngineInput: |
| 488 | arrival_time = time.time() |
| 489 | prompt = TokensPrompt(prompt_token_ids=render_for_completion(messages)) |
| 490 | if tok_params is not None: |
| 491 | tok_params.apply_post_tokenization( |
| 492 | self.renderer.tokenizer, |
| 493 | prompt, |
| 494 | ) |
| 495 | engine_input = tokens_input(prompt["prompt_token_ids"], cache_salt=cache_salt) |
| 496 | engine_input["arrival_time"] = arrival_time |
| 497 | return engine_input |
| 498 | |
| 499 | def _responses_render_result( |
| 500 | self, |
| 501 | messages: list[ChatCompletionMessageParam], |
| 502 | engine_inputs: list[EngineInput], |
| 503 | ) -> ResponsesRenderResult | ErrorResponse: |
| 504 | if len(engine_inputs) != 1: |
| 505 | return self.create_error_response( |
| 506 | f"Expected exactly 1 engine prompt, got {len(engine_inputs)}" |
| 507 | ) |
| 508 | return ResponsesRenderResult( |
| 509 | messages=messages, |
| 510 | engine_input=engine_inputs[0], |
| 511 | ) |
| 512 | |
| 513 | def adjust_harmony_request( |
| 514 | self, request: ChatCompletionRequest | ResponsesRequest |
| 515 | ) -> None: |
| 516 | """Apply the Harmony parser's ``adjust_request`` to ``request`` in place. |
| 517 | |
| 518 | Every Harmony render path must call this before sampling params are |
| 519 | built from ``request``. |
| 520 | """ |
| 521 | if self.parser is None: |
| 522 | return |
| 523 | # HarmonyParser doesn't need chat_template_kwargs |
| 524 | # TODO: Unify adjust_request() call with non-harmony branch |
| 525 | self.parser( |
| 526 | self.renderer.get_tokenizer(), |
| 527 | request.tools, |
| 528 | model_config=self.model_config, |
| 529 | ).adjust_request(request=request) |
| 530 | |
| 531 | def _make_request_with_harmony( |
| 532 | self, |
| 533 | request: ChatCompletionRequest, |
| 534 | should_include_tools: bool = True, |
| 535 | ): |
| 536 | """Build Harmony (GPT-OSS) messages and engine prompt from a chat request.""" |
| 537 | reuse_ids = _reused_prompt_token_ids(request, self.renderer) |
| 538 | if reuse_ids: |
| 539 | # Decode-side token reuse: feed the forwarded ids straight to the |
| 540 | # engine. Harmony has no adjust_request hook to preserve. |
| 541 | engine_input = tokens_input(reuse_ids, cache_salt=request.cache_salt) |
| 542 | return [], [engine_input] |
| 543 | |
| 544 | messages: list[OpenAIMessage] = [] |
| 545 | |
| 546 | # because of issues with pydantic we need to potentially |
| 547 | # re-serialize the tool_calls field of the request |
| 548 | # for more info: see comment in `maybe_serialize_tool_calls` |
| 549 | _mt.maybe_serialize_tool_calls(request) # type: ignore[arg-type] |
| 550 | |
| 551 | chat_messages = list(request.messages) |
| 552 | instructions, chat_messages = extract_instructions_from_messages(chat_messages) |
| 553 | |
| 554 | # Add system message. |
| 555 | # NOTE: In Chat Completion API, browsing is enabled by default |
| 556 | # if the model supports it. TODO: Support browsing. |
| 557 | assert not self.supports_browsing |
| 558 | assert not self.supports_code_interpreter |
| 559 | if (reasoning_effort := request.reasoning_effort) == "none": |
| 560 | raise VLLMValidationError( |
| 561 | f"Harmony does not support {reasoning_effort=}", |
| 562 | parameter="reasoning_effort", |
| 563 | ) |
| 564 | tools = request.tools if should_include_tools else None |
| 565 | messages.extend( |
| 566 | build_harmony_preamble( |
| 567 | instructions=instructions, |
| 568 | tools=tools, # type: ignore[arg-type] |
| 569 | reasoning_effort=reasoning_effort, |
| 570 | with_custom_tools=should_include_tools, |
| 571 | ) |
| 572 | ) |
| 573 | |
| 574 | # Add remaining conversation messages. |
| 575 | messages.extend(parse_chat_inputs_to_harmony_messages(chat_messages)) |
| 576 | |
| 577 | # Render prompt token ids. |
| 578 | prompt_token_ids = render_for_completion(messages) |
| 579 | engine_input = tokens_input(prompt_token_ids, cache_salt=request.cache_salt) |
| 580 | |
| 581 | return messages, [engine_input] |
| 582 | |
| 583 | async def render_completion( |
| 584 | self, |
| 585 | request: CompletionRequest, |
| 586 | *, |
| 587 | skip_mm_cache: bool = False, |
| 588 | ) -> list[EngineInput] | ErrorResponse: |
| 589 | """Core preprocessing logic for completion requests (no model/engine check). |
| 590 | |
| 591 | Called directly by render_completion_request and delegated to by |
| 592 | OpenAIServingCompletion.render_completion_request after its engine-aware checks. |
| 593 | """ |
| 594 | prompt_input = request.prompt |
| 595 | if request.suffix is not None: |
| 596 | if request.echo: |
| 597 | return self.create_error_response( |
| 598 | "Echo is unsupported with suffix.", |
| 599 | param="suffix", |
| 600 | ) |
| 601 | |
| 602 | if request.prompt_embeds is not None: |
| 603 | return self.create_error_response( |
| 604 | "suffix is not supported with prompt_embeds", |
| 605 | param="suffix", |
| 606 | ) |
| 607 | |
| 608 | if request.truncate_prompt_tokens is not None: |
| 609 | return self.create_error_response( |
| 610 | "suffix is not supported with truncate_prompt_tokens", |
| 611 | param="suffix", |
| 612 | ) |
| 613 | |
| 614 | if isinstance(request.prompt, str): |
| 615 | rendered_prompt = self.renderer.render_completion_suffix( |
| 616 | request.prompt, request.suffix |
| 617 | ) |
| 618 | if rendered_prompt is None: |
| 619 | return self.create_error_response( |
| 620 | "suffix is only supported for models with FIM completion " |
| 621 | "rendering", |
| 622 | param="suffix", |
| 623 | ) |
| 624 | prompt_input = rendered_prompt |
| 625 | elif isinstance(request.prompt, list) and all( |
| 626 | isinstance(prompt, str) for prompt in request.prompt |
| 627 | ): |
| 628 | rendered_prompts = [] |
| 629 | for prompt in cast(list[str], request.prompt): |
| 630 | rendered_prompt = self.renderer.render_completion_suffix( |
| 631 | prompt, request.suffix |
| 632 | ) |
| 633 | if rendered_prompt is None: |
| 634 | return self.create_error_response( |
| 635 | "suffix is only supported for models with FIM completion " |
| 636 | "rendering", |
| 637 | param="suffix", |
| 638 | ) |
| 639 | rendered_prompts.append(rendered_prompt) |
| 640 | prompt_input = rendered_prompts |
| 641 | else: |
| 642 | return self.create_error_response( |
| 643 | "suffix requires text prompt input for FIM completion rendering", |
| 644 | param="suffix", |
| 645 | ) |
| 646 | |
| 647 | if request.echo and request.prompt_embeds is not None: |
| 648 | return self.create_error_response("Echo is unsupported with prompt embeds.") |
| 649 | |
| 650 | if request.prompt_logprobs is not None and request.prompt_embeds is not None: |
| 651 | return self.create_error_response( |
| 652 | "prompt_logprobs is not compatible with prompt embeds." |
| 653 | ) |
| 654 | |
| 655 | engine_inputs = await self.preprocess_completion( |
| 656 | request, |
| 657 | prompt_input=prompt_input, |
| 658 | prompt_embeds=request.prompt_embeds, |
| 659 | skip_mm_cache=skip_mm_cache, |
| 660 | ) |
| 661 | |
| 662 | return engine_inputs |
| 663 | |
| 664 | def create_error_response( |
| 665 | self, |
| 666 | message: str | Exception, |
| 667 | err_type: str = "BadRequestError", |
| 668 | status_code: HTTPStatus = HTTPStatus.BAD_REQUEST, |
| 669 | param: str | None = None, |
| 670 | ) -> ErrorResponse: |
| 671 | return create_error_response(message, err_type, status_code, param) |
| 672 | |
| 673 | def validate_chat_template( |
| 674 | self, |
| 675 | request_chat_template: str | None, |
| 676 | chat_template_kwargs: dict[str, Any] | None, |
| 677 | trust_request_chat_template: bool, |
| 678 | ) -> ErrorResponse | None: |
| 679 | """Copied from GenerateBaseServing._validate_chat_template.""" |
| 680 | if not trust_request_chat_template and ( |
| 681 | request_chat_template is not None |
| 682 | or ( |
| 683 | chat_template_kwargs |
| 684 | and chat_template_kwargs.get("chat_template") is not None |
| 685 | ) |
| 686 | ): |
| 687 | return self.create_error_response( |
| 688 | "Chat template is passed with request, but " |
| 689 | "--trust-request-chat-template is not set. " |
| 690 | "Refused request with untrusted chat template." |
| 691 | ) |
| 692 | return None |
| 693 | |
| 694 | async def preprocess_completion( |
| 695 | self, |
| 696 | request: Any, |
| 697 | prompt_input: str | list[str] | list[int] | list[list[int]] | None, |
| 698 | prompt_embeds: bytes | list[bytes] | None, |
| 699 | *, |
| 700 | skip_mm_cache: bool = False, |
| 701 | ) -> list[EngineInput]: |
| 702 | """Copied from GenerateBaseServing._preprocess_completion.""" |
| 703 | prompts = list[SingletonPrompt | bytes]() |
| 704 | if prompt_embeds is not None: # embeds take higher priority |
| 705 | prompts.extend(prompt_to_seq(prompt_embeds)) |
| 706 | if prompt_input is not None: |
| 707 | prompts.extend(prompt_to_seq(prompt_input)) |
| 708 | return await self.preprocess_cmpl(request, prompts, skip_mm_cache=skip_mm_cache) |
| 709 | |
| 710 | async def preprocess_cmpl( |
| 711 | self, |
| 712 | request: Any, |
| 713 | prompts: Sequence[PromptType | bytes], |
| 714 | *, |
| 715 | skip_mm_cache: bool = False, |
| 716 | ) -> list[EngineInput]: |
| 717 | """Copied from GenerateBaseServing._preprocess_cmpl.""" |
| 718 | renderer = self.renderer |
| 719 | model_config = self.model_config |
| 720 | validate_request_mm_kwargs( |
| 721 | mm_processor_kwargs=getattr(request, "mm_processor_kwargs", None), |
| 722 | media_io_kwargs=getattr(request, "media_io_kwargs", None), |
| 723 | trust_request_mm_kwargs=self.trust_request_mm_kwargs, |
| 724 | ) |
| 725 | |
| 726 | parsed_prompts = [ |
| 727 | ( |
| 728 | prompt |
| 729 | if isinstance(prompt, bytes) |
| 730 | else parse_model_prompt(model_config, prompt) |
| 731 | ) |
| 732 | for prompt in prompts |
| 733 | ] |
| 734 | tok_params = request.build_tok_params(model_config) |
| 735 | |
| 736 | return await renderer.render_cmpl_async( |
| 737 | parsed_prompts, |
| 738 | tok_params, |
| 739 | prompt_extras={ |
| 740 | k: v |
| 741 | for k in ("mm_processor_kwargs", "cache_salt") |
| 742 | if (v := getattr(request, k, None)) is not None |
| 743 | }, |
| 744 | skip_mm_cache=skip_mm_cache, |
| 745 | ) |
| 746 | |
| 747 | async def preprocess_chat( |
| 748 | self, |
| 749 | request: Any, |
| 750 | messages: list[Any], |
| 751 | default_template: str | None, |
| 752 | default_template_content_format: ChatTemplateContentFormatOption, |
| 753 | default_template_kwargs: dict[str, Any] | None, |
| 754 | tool_dicts: list[dict[str, Any]] | None = None, |
| 755 | parser: type[Parser] | None = None, |
| 756 | *, |
| 757 | skip_mm_cache: bool = False, |
| 758 | ) -> tuple[list[ConversationMessage], list[EngineInput]]: |
| 759 | """Copied from GenerateBaseServing._preprocess_chat.""" |
| 760 | renderer = self.renderer |
| 761 | validate_request_mm_kwargs( |
| 762 | mm_processor_kwargs=getattr(request, "mm_processor_kwargs", None), |
| 763 | media_io_kwargs=getattr(request, "media_io_kwargs", None), |
| 764 | trust_request_mm_kwargs=self.trust_request_mm_kwargs, |
| 765 | ) |
| 766 | mm_config = self.model_config.multimodal_config |
| 767 | |
| 768 | default_template_kwargs = merge_kwargs( |
| 769 | default_template_kwargs, |
| 770 | dict( |
| 771 | tools=tool_dicts, |
| 772 | tokenize=( |
| 773 | is_mistral_tokenizer(renderer.tokenizer) |
| 774 | or self.model_config.enable_prompt_embeds |
| 775 | ), |
| 776 | ), |
| 777 | ) |
| 778 | |
| 779 | tok_params = request.build_tok_params(self.model_config) |
| 780 | chat_params = request.build_chat_params( |
| 781 | default_template, default_template_content_format |
| 782 | ).with_defaults( |
| 783 | default_template_kwargs, |
| 784 | default_media_io_kwargs=(mm_config.media_io_kwargs if mm_config else None), |
| 785 | default_mm_processor_kwargs=getattr(request, "mm_processor_kwargs", None), |
| 786 | ) |
| 787 | |
| 788 | reuse_ids = _reused_prompt_token_ids(request, renderer, messages) |
| 789 | if reuse_ids: |
| 790 | # Decode-side token reuse: feed the forwarded ids straight to the |
| 791 | # engine, skipping templating and tokenization. ``messages`` are not |
| 792 | # tokenized, so conversation is empty. The adjust_request tail below |
| 793 | # still runs. |
| 794 | conversation: list[ConversationMessage] = [] |
| 795 | engine_input = tokens_input( |
| 796 | reuse_ids, cache_salt=getattr(request, "cache_salt", None) |
| 797 | ) |
| 798 | else: |
| 799 | (conversation,), (engine_input,) = await renderer.render_chat_async( |
| 800 | [messages], |
| 801 | chat_params, |
| 802 | tok_params, |
| 803 | prompt_extras={ |
| 804 | k: v |
| 805 | for k in ("mm_processor_kwargs", "cache_salt") |
| 806 | if (v := getattr(request, k, None)) is not None |
| 807 | }, |
| 808 | skip_mm_cache=skip_mm_cache, |
| 809 | ) |
| 810 | |
| 811 | # tool parsing is done only if a tool_parser has been set and if |
| 812 | # tool_choice is not "none" (if tool_choice is "none" but a tool_parser |
| 813 | # is set, we want to prevent parsing a tool_call hallucinated by the LLM |
| 814 | # |
| 815 | # Exception: Mistral grammar-capable tokenizers always call |
| 816 | # adjust_request — even for tool_choice="none" — so that the grammar |
| 817 | # factory can prevent special-token leakage. |
| 818 | if parser is not None: |
| 819 | tokenizer = renderer.get_tokenizer() |
| 820 | tool_parser = parser.tool_parser_cls |
| 821 | tool_choice = getattr(request, "tool_choice", "none") |
| 822 | is_mistral_grammar_eligible = ( |
| 823 | tool_parser is not None |
| 824 | and is_mistral_tool_parser(tool_parser) |
| 825 | and is_mistral_tokenizer(tokenizer) |
| 826 | and tokenizer.supports_grammar |
| 827 | ) |
| 828 | should_adjust_request = ( |
| 829 | parser.always_adjust_request |
| 830 | or parser.reasoning_parser_cls is not None |
| 831 | or tool_choice != "none" |
| 832 | or is_mistral_grammar_eligible |
| 833 | ) |
| 834 | if should_adjust_request: |
| 835 | if not isinstance(request, ChatCompletionRequest | ResponsesRequest): |
| 836 | msg = ( |
| 837 | "Tool usage is only supported " |
| 838 | "for Chat Completions API or Responses API requests, " |
| 839 | f"but got {type(request).__name__}" |
| 840 | ) |
| 841 | raise NotImplementedError(msg) |
| 842 | request = parser( |
| 843 | tokenizer, |
| 844 | request.tools, |
| 845 | model_config=self.model_config, |
| 846 | chat_template_kwargs=chat_params.chat_template_kwargs, |
| 847 | ).adjust_request( |
| 848 | request=request, |
| 849 | ) |
| 850 | |
| 851 | return conversation, [engine_input] |