| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import asyncio |
| 5 | import time |
| 6 | from collections.abc import AsyncGenerator, AsyncIterator |
| 7 | from collections.abc import Sequence as GenericSequence |
| 8 | from http import HTTPStatus |
| 9 | from typing import Any, Final, cast |
| 10 | |
| 11 | from fastapi import Request |
| 12 | |
| 13 | from vllm.engine.protocol import EngineClient |
| 14 | from vllm.entrypoints.chat_utils import ( |
| 15 | ChatTemplateContentFormatOption, |
| 16 | ConversationMessage, |
| 17 | make_tool_call_id, |
| 18 | ) |
| 19 | from vllm.entrypoints.generate.base.protocol import ( |
| 20 | DeltaMessage, |
| 21 | FunctionCall, |
| 22 | PerRequestMetrics, |
| 23 | RequestResponseMetadata, |
| 24 | ToolCall, |
| 25 | ) |
| 26 | from vllm.entrypoints.generate.base.serving import ( |
| 27 | GenerateBaseServing, |
| 28 | build_per_request_timing_metrics, |
| 29 | build_spec_decoding_metrics, |
| 30 | clamp_prompt_logprobs, |
| 31 | format_token_id_placeholder, |
| 32 | ) |
| 33 | from vllm.entrypoints.openai.chat_completion.protocol import ( |
| 34 | ChatCompletionLogProb, |
| 35 | ChatCompletionLogProbs, |
| 36 | ChatCompletionLogProbsContent, |
| 37 | ChatCompletionNamedToolChoiceParam, |
| 38 | ChatCompletionRequest, |
| 39 | ChatCompletionResponse, |
| 40 | ChatCompletionResponseChoice, |
| 41 | ChatCompletionResponseStreamChoice, |
| 42 | ChatCompletionStreamResponse, |
| 43 | ChatMessage, |
| 44 | ) |
| 45 | from vllm.entrypoints.openai.models.serving import OpenAIServingModels |
| 46 | from vllm.entrypoints.serve.engine.protocol import ( |
| 47 | CompletionTokenUsageInfo, |
| 48 | ErrorResponse, |
| 49 | PromptTokenUsageInfo, |
| 50 | UsageInfo, |
| 51 | ) |
| 52 | from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage |
| 53 | from vllm.entrypoints.serve.utils.request_logger import RequestLogger |
| 54 | from vllm.entrypoints.serve.utils.tool_calls_utils import ( |
| 55 | maybe_filter_parallel_tool_calls, |
| 56 | ) |
| 57 | from vllm.exceptions import GenerationError |
| 58 | from vllm.inputs import EngineInput, MultiModalPlaceholders |
| 59 | from vllm.logger import init_logger |
| 60 | from vllm.logprobs import Logprob |
| 61 | from vllm.outputs import RequestOutput |
| 62 | from vllm.parser import ParserManager |
| 63 | from vllm.parser.abstract_parser import Parser |
| 64 | from vllm.renderers.online_renderer import OnlineRenderer |
| 65 | from vllm.sampling_params import BeamSearchParams, SamplingParams |
| 66 | from vllm.tokenizers import TokenizerLike |
| 67 | from vllm.utils.collection_utils import as_list |
| 68 | from vllm.utils.serial_utils import numpy2base64 |
| 69 | |
| 70 | logger = init_logger(__name__) |
| 71 | |
| 72 | |
| 73 | def _get_mm_token_counts(engine_input: EngineInput) -> dict[str, int]: |
| 74 | """Sum per-modality placeholder tokens from ``mm_placeholders``. |
| 75 | |
| 76 | Keyed by modality name; ``PlaceholderRange.length`` is the placeholder's |
| 77 | prompt token span, so each sum matches the placeholder tokens already |
| 78 | counted in ``usage.prompt_tokens``. |
| 79 | """ |
| 80 | mm_placeholders = cast( |
| 81 | MultiModalPlaceholders | None, engine_input.get("mm_placeholders") |
| 82 | ) |
| 83 | return { |
| 84 | modality: sum(p.length for p in ranges) |
| 85 | for modality, ranges in (mm_placeholders or {}).items() |
| 86 | if ranges |
| 87 | } |
| 88 | |
| 89 | |
| 90 | def _make_prompt_tokens_details( |
| 91 | enable_prompt_tokens_details: bool, |
| 92 | num_cached_tokens: int | None, |
| 93 | num_cache_creation_tokens: int | None, |
| 94 | mm_token_counts: dict[str, int] | None, |
| 95 | ) -> PromptTokenUsageInfo | None: |
| 96 | """Build ``prompt_tokens_details`` from cached + multimodal token counts.""" |
| 97 | if not enable_prompt_tokens_details: |
| 98 | return None |
| 99 | if ( |
| 100 | num_cached_tokens is None |
| 101 | and num_cache_creation_tokens is None |
| 102 | and not mm_token_counts |
| 103 | ): |
| 104 | return None |
| 105 | return PromptTokenUsageInfo( |
| 106 | cached_tokens=num_cached_tokens, |
| 107 | created_cache_tokens=num_cache_creation_tokens, |
| 108 | multimodal_tokens=mm_token_counts or None, |
| 109 | ) |
| 110 | |
| 111 | |
| 112 | def _make_completion_tokens_details( |
| 113 | reasoning_tokens: int, |
| 114 | ) -> CompletionTokenUsageInfo: |
| 115 | return CompletionTokenUsageInfo(reasoning_tokens=reasoning_tokens) |
| 116 | |
| 117 | |
| 118 | class OpenAIServingChat(GenerateBaseServing): |
| 119 | def __init__( |
| 120 | self, |
| 121 | engine_client: EngineClient, |
| 122 | models: OpenAIServingModels, |
| 123 | response_role: str, |
| 124 | *, |
| 125 | online_renderer: "OnlineRenderer", |
| 126 | request_logger: RequestLogger | None, |
| 127 | chat_template: str | None, |
| 128 | chat_template_content_format: ChatTemplateContentFormatOption, |
| 129 | trust_request_chat_template: bool = False, |
| 130 | return_tokens_as_token_ids: bool = False, |
| 131 | reasoning_parser: str = "", |
| 132 | enable_auto_tools: bool = False, |
| 133 | exclude_tools_when_tool_choice_none: bool = False, |
| 134 | tool_parser: str | None = None, |
| 135 | tool_strict_level: str = "auto", |
| 136 | enable_prompt_tokens_details: bool = False, |
| 137 | enable_force_include_usage: bool = False, |
| 138 | enable_log_outputs: bool = False, |
| 139 | enable_log_deltas: bool = True, |
| 140 | default_chat_template_kwargs: dict[str, Any] | None = None, |
| 141 | enable_per_request_metrics: bool = False, |
| 142 | ) -> None: |
| 143 | super().__init__( |
| 144 | engine_client=engine_client, |
| 145 | models=models, |
| 146 | request_logger=request_logger, |
| 147 | return_tokens_as_token_ids=return_tokens_as_token_ids, |
| 148 | ) |
| 149 | |
| 150 | self.online_renderer = online_renderer |
| 151 | self.response_role = response_role |
| 152 | self.chat_template = chat_template |
| 153 | self.chat_template_content_format: Final = chat_template_content_format |
| 154 | self.trust_request_chat_template = trust_request_chat_template |
| 155 | self.default_chat_template_kwargs = default_chat_template_kwargs or {} |
| 156 | self.enable_log_outputs = enable_log_outputs |
| 157 | self.enable_log_deltas = enable_log_deltas |
| 158 | |
| 159 | self.enable_auto_tools: bool = enable_auto_tools |
| 160 | self._include_reasoning_tokens_details = bool(reasoning_parser) |
| 161 | self.parser_cls = ParserManager.get_parser( |
| 162 | tool_parser_name=tool_parser, |
| 163 | reasoning_parser_name=reasoning_parser, |
| 164 | enable_auto_tools=enable_auto_tools, |
| 165 | tool_strict_level=tool_strict_level, |
| 166 | model_name=self.model_config.model, |
| 167 | is_harmony=self.model_config.hf_config.model_type == "gpt_oss", |
| 168 | tokenizer=self.renderer.tokenizer, |
| 169 | ) |
| 170 | self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none |
| 171 | |
| 172 | self.enable_prompt_tokens_details = enable_prompt_tokens_details |
| 173 | self.enable_force_include_usage = enable_force_include_usage |
| 174 | self.enable_per_request_metrics = enable_per_request_metrics |
| 175 | self.default_sampling_params = self.model_config.get_diff_sampling_param() |
| 176 | mc = self.model_config |
| 177 | self.override_max_tokens = ( |
| 178 | self.default_sampling_params.get("max_tokens") |
| 179 | if mc.generation_config not in ("auto", "vllm") |
| 180 | else getattr(mc, "override_generation_config", {}).get("max_new_tokens") |
| 181 | ) |
| 182 | # NOTE(woosuk): While OpenAI's chat completion API supports browsing |
| 183 | # for some models, currently vLLM doesn't support it. Please use the |
| 184 | # Responses API instead. |
| 185 | self.supports_browsing = False |
| 186 | self.browser_tool = None |
| 187 | # NOTE(woosuk): Chat completion API does not support code interpreter. |
| 188 | # Please use the Responses API instead. |
| 189 | self.supports_code_interpreter = False |
| 190 | self.python_tool = None |
| 191 | |
| 192 | def _effective_chat_template_kwargs( |
| 193 | self, request: ChatCompletionRequest |
| 194 | ) -> dict[str, Any]: |
| 195 | return ( |
| 196 | request.build_chat_params( |
| 197 | self.chat_template, |
| 198 | self.chat_template_content_format, |
| 199 | ) |
| 200 | .with_defaults(self.default_chat_template_kwargs) |
| 201 | .chat_template_kwargs |
| 202 | ) |
| 203 | |
| 204 | def _make_parser( |
| 205 | self, |
| 206 | request: ChatCompletionRequest, |
| 207 | tokenizer: TokenizerLike, |
| 208 | chat_template_kwargs: dict[str, Any] | None, |
| 209 | ) -> Parser: |
| 210 | assert self.parser_cls is not None |
| 211 | return self.parser_cls( |
| 212 | tokenizer, |
| 213 | request.tools, |
| 214 | chat_template_kwargs=chat_template_kwargs, |
| 215 | model_config=self.model_config, |
| 216 | ) |
| 217 | |
| 218 | def _engine_chat_template_kwargs( |
| 219 | self, chat_template_kwargs: dict[str, Any] |
| 220 | ) -> dict[str, Any]: |
| 221 | """Subclass hook to narrow ``chat_template_kwargs`` for the engine. |
| 222 | |
| 223 | The same dict is used twice: to build the API-server-side parser |
| 224 | instance, and to populate |
| 225 | ``EngineCoreRequest.reasoning_parser_kwargs`` for the engine-core |
| 226 | side. The latter crosses ZMQ as msgpack, so a handler that stashes |
| 227 | request-scoped state only the API-server-side parser needs (values |
| 228 | msgpack can't encode, or payloads not worth shipping) can drop |
| 229 | those entries here without affecting the in-process parser. |
| 230 | |
| 231 | Must not mutate the argument -- the caller still needs the full |
| 232 | dict. The default forwards it unchanged. |
| 233 | """ |
| 234 | return chat_template_kwargs |
| 235 | |
| 236 | async def render_chat_request( |
| 237 | self, |
| 238 | request: ChatCompletionRequest, |
| 239 | ) -> tuple[list[ConversationMessage], list[EngineInput]] | ErrorResponse: |
| 240 | """Validate the model and preprocess a chat completion request. |
| 241 | |
| 242 | Delegates preprocessing logic to OnlineRenderer, adding the |
| 243 | engine-aware checks (LoRA model validation, engine health). |
| 244 | |
| 245 | Returns: |
| 246 | A tuple of (conversation, engine_inputs) on success, |
| 247 | or an ErrorResponse on failure. |
| 248 | |
| 249 | """ |
| 250 | error_check_ret = await self._check_model(request) |
| 251 | if error_check_ret is not None: |
| 252 | logger.error("Error with model %s", error_check_ret) |
| 253 | return error_check_ret |
| 254 | |
| 255 | self._preflight(request.n or 1) |
| 256 | |
| 257 | return await self.online_renderer.render_chat(request) |
| 258 | |
| 259 | async def create_chat_completion( |
| 260 | self, |
| 261 | request: ChatCompletionRequest, |
| 262 | raw_request: Request | None = None, |
| 263 | ) -> AsyncGenerator[str, None] | ChatCompletionResponse | ErrorResponse: |
| 264 | """Chat Completion API similar to OpenAI's API. |
| 265 | |
| 266 | See https://platform.openai.com/docs/api-reference/chat/create |
| 267 | for the API specification. This API mimics the OpenAI |
| 268 | Chat Completion API. |
| 269 | """ |
| 270 | return await self._with_kv_transfer_rejection_cleanup( |
| 271 | self._create_chat_completion(request, raw_request), request, raw_request |
| 272 | ) |
| 273 | |
| 274 | async def _create_chat_completion( |
| 275 | self, |
| 276 | request: ChatCompletionRequest, |
| 277 | raw_request: Request | None = None, |
| 278 | ) -> AsyncGenerator[str, None] | ChatCompletionResponse | ErrorResponse: |
| 279 | # Streaming response |
| 280 | tokenizer = self.renderer.tokenizer |
| 281 | assert tokenizer is not None |
| 282 | chat_template_kwargs = self._effective_chat_template_kwargs(request) |
| 283 | parser: Parser | None = None |
| 284 | if self.parser_cls is not None: |
| 285 | parser = self._make_parser(request, tokenizer, chat_template_kwargs) |
| 286 | result = await self.render_chat_request(request) |
| 287 | if isinstance(result, ErrorResponse): |
| 288 | return result |
| 289 | |
| 290 | conversation, engine_inputs = result |
| 291 | |
| 292 | request_id = ( |
| 293 | f"chatcmpl-{self._base_request_id(raw_request, request.request_id)}" |
| 294 | ) |
| 295 | |
| 296 | request_metadata = RequestResponseMetadata(request_id=request_id) |
| 297 | if raw_request: |
| 298 | raw_request.state.request_metadata = request_metadata |
| 299 | |
| 300 | lora_request = self._maybe_get_adapters(request, supports_default_mm_loras=True) |
| 301 | |
| 302 | model_name = self.models.model_name(lora_request) |
| 303 | |
| 304 | # Extract data_parallel_rank from header (router can inject it) |
| 305 | data_parallel_rank = self._get_data_parallel_rank(raw_request) |
| 306 | |
| 307 | # Schedule the request and get the result generator. |
| 308 | max_model_len = self.model_config.max_model_len |
| 309 | generators: list[AsyncGenerator[RequestOutput, None]] = [] |
| 310 | mm_token_counts: dict[str, int] | None = None |
| 311 | for i, engine_input in enumerate(engine_inputs): |
| 312 | prompt_token_ids = self._extract_prompt_components(engine_input).token_ids |
| 313 | mm_token_counts = _get_mm_token_counts(engine_input) |
| 314 | |
| 315 | # If we are creating sub requests for multiple prompts, ensure that they |
| 316 | # have unique request ids. |
| 317 | sub_request_id = ( |
| 318 | request_id if len(engine_inputs) == 1 else f"{request_id}_{i}" |
| 319 | ) |
| 320 | |
| 321 | max_tokens = get_max_tokens( |
| 322 | max_model_len, |
| 323 | request.max_completion_tokens |
| 324 | if request.max_completion_tokens is not None |
| 325 | else request.max_tokens, |
| 326 | self._extract_prompt_len(engine_input), |
| 327 | self.default_sampling_params, |
| 328 | self.override_max_tokens, |
| 329 | truncate_prompt_tokens=request.truncate_prompt_tokens, |
| 330 | ) |
| 331 | |
| 332 | sampling_params: SamplingParams | BeamSearchParams |
| 333 | if request.use_beam_search: |
| 334 | sampling_params = request.to_beam_search_params( |
| 335 | max_tokens, self.default_sampling_params |
| 336 | ) |
| 337 | else: |
| 338 | sampling_params = request.to_sampling_params( |
| 339 | max_tokens, |
| 340 | self.default_sampling_params, |
| 341 | ) |
| 342 | |
| 343 | self._log_inputs( |
| 344 | sub_request_id, |
| 345 | engine_input, |
| 346 | params=sampling_params, |
| 347 | lora_request=lora_request, |
| 348 | ) |
| 349 | |
| 350 | trace_headers = ( |
| 351 | None |
| 352 | if raw_request is None |
| 353 | else await self._get_trace_headers(raw_request.headers) |
| 354 | ) |
| 355 | session_id = self._get_session_id(request, raw_request) |
| 356 | |
| 357 | if isinstance(sampling_params, BeamSearchParams): |
| 358 | generator = self.beam_search( |
| 359 | prompt=engine_input, |
| 360 | request_id=sub_request_id, |
| 361 | params=sampling_params, |
| 362 | lora_request=lora_request, |
| 363 | trace_headers=trace_headers, |
| 364 | session_id=session_id, |
| 365 | ) |
| 366 | else: |
| 367 | if not request.include_reasoning: |
| 368 | reasoning_ended = True |
| 369 | elif request._grammar_from_parser: |
| 370 | # The Mistral grammar already includes an optional |
| 371 | # `think?` rule that handles both reasoning and |
| 372 | # non-reasoning outputs. |
| 373 | reasoning_ended = True |
| 374 | elif parser is not None and parser.reasoning_parser is not None: |
| 375 | reasoning_ended = parser.is_reasoning_end(prompt_token_ids or []) |
| 376 | else: |
| 377 | reasoning_ended = None |
| 378 | |
| 379 | generator = self.engine_client.generate( |
| 380 | engine_input, |
| 381 | sampling_params, |
| 382 | sub_request_id, |
| 383 | lora_request=lora_request, |
| 384 | trace_headers=trace_headers, |
| 385 | priority=self._get_priority(request, raw_request), |
| 386 | data_parallel_rank=data_parallel_rank, |
| 387 | session_id=session_id, |
| 388 | reasoning_ended=reasoning_ended, |
| 389 | reasoning_parser_kwargs={ |
| 390 | "chat_template_kwargs": self._engine_chat_template_kwargs( |
| 391 | chat_template_kwargs |
| 392 | ), |
| 393 | } |
| 394 | if parser is not None and parser.reasoning_parser is not None |
| 395 | else None, |
| 396 | ) |
| 397 | |
| 398 | generators.append(generator) |
| 399 | |
| 400 | assert len(generators) == 1 |
| 401 | (result_generator,) = generators |
| 402 | |
| 403 | if request.stream: |
| 404 | return self.chat_completion_stream_generator( |
| 405 | request, |
| 406 | result_generator, |
| 407 | request_id, |
| 408 | model_name, |
| 409 | conversation, |
| 410 | tokenizer, |
| 411 | request_metadata, |
| 412 | chat_template_kwargs=chat_template_kwargs, |
| 413 | mm_token_counts=mm_token_counts, |
| 414 | ) |
| 415 | |
| 416 | return await self.chat_completion_full_generator( |
| 417 | request, |
| 418 | result_generator, |
| 419 | request_id, |
| 420 | model_name, |
| 421 | conversation, |
| 422 | tokenizer, |
| 423 | request_metadata, |
| 424 | parser=parser, |
| 425 | mm_token_counts=mm_token_counts, |
| 426 | ) |
| 427 | |
| 428 | def get_chat_request_role(self, request: ChatCompletionRequest) -> str: |
| 429 | if request.add_generation_prompt: |
| 430 | return self.response_role |
| 431 | return request.messages[-1]["role"] |
| 432 | |
| 433 | def _create_chat_message(self, *args: Any, **kwargs: Any) -> ChatMessage: |
| 434 | """Construct the response :class:`ChatMessage` for the non-streaming path. |
| 435 | |
| 436 | The full-generator calls this at every construction site so |
| 437 | subclasses can swap in a specialized :class:`ChatMessage` |
| 438 | subclass (e.g. :class:`CohereServingChatV2` returning |
| 439 | :class:`CohereChatMessage`) without duplicating the branchy |
| 440 | tool-choice / auto-tools logic that decides which fields are |
| 441 | populated. The default returns a plain :class:`ChatMessage`. |
| 442 | """ |
| 443 | return ChatMessage(*args, **kwargs) |
| 444 | |
| 445 | def _finalize_response_message( |
| 446 | self, |
| 447 | message: ChatMessage, |
| 448 | *, |
| 449 | parser: Parser | None, |
| 450 | ) -> ChatMessage: |
| 451 | """Subclass hook to enrich a fully-constructed :class:`ChatMessage`. |
| 452 | |
| 453 | Default is a no-op. Subclasses that need to surface parser-side |
| 454 | extras (e.g. :class:`CohereServingChatV2` reading grounding |
| 455 | citations off the reasoning parser and populating |
| 456 | :class:`CohereChatMessage.citations`) override this to inspect |
| 457 | ``parser`` and mutate/replace ``message``. |
| 458 | """ |
| 459 | return message |
| 460 | |
| 461 | async def chat_completion_stream_generator( |
| 462 | self, |
| 463 | request: ChatCompletionRequest, |
| 464 | result_generator: AsyncIterator[RequestOutput], |
| 465 | request_id: str, |
| 466 | model_name: str, |
| 467 | conversation: list[ConversationMessage], |
| 468 | tokenizer: TokenizerLike, |
| 469 | request_metadata: RequestResponseMetadata, |
| 470 | chat_template_kwargs: dict[str, Any] | None = None, |
| 471 | mm_token_counts: dict[str, int] | None = None, |
| 472 | ) -> AsyncGenerator[str, None]: |
| 473 | created_time = int(time.time()) |
| 474 | chunk_object_type: Final = "chat.completion.chunk" |
| 475 | first_iteration = True |
| 476 | |
| 477 | # Send response for each token for each request.n (index) |
| 478 | num_choices = 1 if request.n is None else request.n |
| 479 | previous_num_tokens = [0] * num_choices |
| 480 | # TODO: Remove once all reasoning parsers use the Parser Engine. |
| 481 | generated_token_ids: list[list[int]] = [[] for _ in range(num_choices)] |
| 482 | previous_reasoning_tokens = [0] * num_choices |
| 483 | finish_reason_sent = [False] * num_choices |
| 484 | num_prompt_tokens = 0 |
| 485 | num_cached_tokens = None |
| 486 | num_cache_creation_tokens = None |
| 487 | tools_streamed = [False] * num_choices |
| 488 | |
| 489 | if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam): |
| 490 | tool_choice_function_name = request.tool_choice.function.name |
| 491 | else: |
| 492 | tool_choice_function_name = None |
| 493 | |
| 494 | previous_texts = [""] * num_choices |
| 495 | |
| 496 | try: |
| 497 | if self.parser_cls is not None: |
| 498 | if tokenizer is None: |
| 499 | raise ValueError( |
| 500 | "Tokenizer not available when `skip_tokenizer_init=True`" |
| 501 | ) |
| 502 | parsers: list[Parser | None] = [ |
| 503 | self._make_parser(request, tokenizer, chat_template_kwargs) |
| 504 | for _ in range(num_choices) |
| 505 | ] |
| 506 | else: |
| 507 | parsers = [None] * num_choices |
| 508 | except Exception as e: |
| 509 | logger.exception("Error in parser creation.") |
| 510 | data = self.create_streaming_error_response(e) |
| 511 | yield f"data: {data}\n\n" |
| 512 | yield "data: [DONE]\n\n" |
| 513 | return |
| 514 | |
| 515 | stream_options = request.stream_options |
| 516 | include_usage, include_continuous_usage = should_include_usage( |
| 517 | stream_options, self.enable_force_include_usage |
| 518 | ) |
| 519 | |
| 520 | last_res: RequestOutput | None = None |
| 521 | try: |
| 522 | async for res in result_generator: |
| 523 | last_res = res |
| 524 | if res.prompt_token_ids is not None: |
| 525 | num_prompt_tokens = len(res.prompt_token_ids) |
| 526 | if res.encoder_prompt_token_ids is not None: |
| 527 | num_prompt_tokens += len(res.encoder_prompt_token_ids) |
| 528 | |
| 529 | # We need to do it here, because if there are exceptions in |
| 530 | # the result_generator, it needs to be sent as the FIRST |
| 531 | # response (by the try...catch). |
| 532 | if first_iteration: |
| 533 | num_cached_tokens = res.num_cached_tokens |
| 534 | num_cache_creation_tokens = res.num_cache_creation_tokens |
| 535 | # Send first response for each request.n (index) with |
| 536 | # the role |
| 537 | role = self.get_chat_request_role(request) |
| 538 | |
| 539 | # ``res.prompt`` is the rendered chat-templated prompt |
| 540 | prompt_text = res.prompt if request.return_prompt_text else None |
| 541 | |
| 542 | # NOTE num_choices defaults to 1 so this usually executes |
| 543 | # once per request |
| 544 | for i in range(num_choices): |
| 545 | choice_data = ChatCompletionResponseStreamChoice( |
| 546 | index=i, |
| 547 | delta=DeltaMessage( |
| 548 | role=role, |
| 549 | content="", |
| 550 | ), |
| 551 | logprobs=None, |
| 552 | finish_reason=None, |
| 553 | ) |
| 554 | |
| 555 | # return prompt_token_ids at the first chunk ever |
| 556 | chunk = ChatCompletionStreamResponse( |
| 557 | id=request_id, |
| 558 | object=chunk_object_type, |
| 559 | created=created_time, |
| 560 | choices=[choice_data], |
| 561 | model=model_name, |
| 562 | prompt_token_ids=( |
| 563 | res.prompt_token_ids |
| 564 | if request.return_token_ids |
| 565 | else None |
| 566 | ), |
| 567 | prompt_text=prompt_text, |
| 568 | ) |
| 569 | |
| 570 | # if continuous usage stats are requested, add it |
| 571 | if include_continuous_usage: |
| 572 | chunk.usage = UsageInfo( |
| 573 | prompt_tokens=num_prompt_tokens, |
| 574 | completion_tokens=0, |
| 575 | total_tokens=num_prompt_tokens, |
| 576 | completion_tokens_details=( |
| 577 | _make_completion_tokens_details(0) |
| 578 | if self._include_reasoning_tokens_details |
| 579 | else None |
| 580 | ), |
| 581 | ) |
| 582 | |
| 583 | data = chunk.model_dump_json(exclude_unset=True) |
| 584 | yield f"data: {data}\n\n" |
| 585 | |
| 586 | # Send response to echo the input portion of the |
| 587 | # last message |
| 588 | if request.echo: |
| 589 | last_msg_content: str | list[dict[str, str]] = "" |
| 590 | if ( |
| 591 | conversation |
| 592 | and "content" in conversation[-1] |
| 593 | and conversation[-1].get("role") == role |
| 594 | ): |
| 595 | last_msg_content = conversation[-1]["content"] or "" |
| 596 | |
| 597 | if last_msg_content: |
| 598 | for i in range(num_choices): |
| 599 | choice_data = ChatCompletionResponseStreamChoice( |
| 600 | index=i, |
| 601 | delta=DeltaMessage(content=last_msg_content), |
| 602 | logprobs=None, |
| 603 | finish_reason=None, |
| 604 | ) |
| 605 | chunk = ChatCompletionStreamResponse( |
| 606 | id=request_id, |
| 607 | object=chunk_object_type, |
| 608 | created=created_time, |
| 609 | choices=[choice_data], |
| 610 | model=model_name, |
| 611 | ) |
| 612 | if include_continuous_usage: |
| 613 | chunk.usage = UsageInfo( |
| 614 | prompt_tokens=num_prompt_tokens, |
| 615 | completion_tokens=0, |
| 616 | total_tokens=num_prompt_tokens, |
| 617 | completion_tokens_details=( |
| 618 | _make_completion_tokens_details(0) |
| 619 | if self._include_reasoning_tokens_details |
| 620 | else None |
| 621 | ), |
| 622 | ) |
| 623 | |
| 624 | data = chunk.model_dump_json(exclude_unset=True) |
| 625 | yield f"data: {data}\n\n" |
| 626 | first_iteration = False |
| 627 | |
| 628 | for output in res.outputs: |
| 629 | i = output.index |
| 630 | parser = parsers[i] |
| 631 | if finish_reason_sent[i]: |
| 632 | continue |
| 633 | |
| 634 | if request.logprobs and ( |
| 635 | request.top_logprobs is not None or request.logprob_token_ids |
| 636 | ): |
| 637 | assert output.logprobs is not None, "Did not output logprobs" |
| 638 | logprobs = self._create_chat_logprobs( |
| 639 | token_ids=output.token_ids, |
| 640 | top_logprobs=output.logprobs, |
| 641 | tokenizer=tokenizer, |
| 642 | num_output_top_logprobs=request.top_logprobs, |
| 643 | logprob_token_ids=request.logprob_token_ids, |
| 644 | return_as_token_id=request.return_tokens_as_token_ids, |
| 645 | ) |
| 646 | else: |
| 647 | logprobs = None |
| 648 | |
| 649 | delta_text = output.text |
| 650 | |
| 651 | if ( |
| 652 | not delta_text |
| 653 | and not output.token_ids |
| 654 | and not previous_num_tokens[i] |
| 655 | ): |
| 656 | # Chunked prefill case, don't return empty chunks |
| 657 | continue |
| 658 | |
| 659 | delta_message: DeltaMessage | None |
| 660 | |
| 661 | if parser is not None: |
| 662 | delta_message = parser.parse_delta( |
| 663 | delta_text=delta_text, |
| 664 | delta_token_ids=as_list(output.token_ids), |
| 665 | request=request, |
| 666 | prompt_token_ids=res.prompt_token_ids, |
| 667 | finished=output.finish_reason is not None, |
| 668 | ) |
| 669 | if delta_message is not None and delta_message.tool_calls: |
| 670 | tools_streamed[i] = True |
| 671 | |
| 672 | # handle streaming just a content delta (no parsers) |
| 673 | else: |
| 674 | delta_message = DeltaMessage(content=delta_text) |
| 675 | |
| 676 | previous_texts[i] += delta_text |
| 677 | |
| 678 | # set the previous values for the next iteration |
| 679 | previous_num_tokens[i] += len(output.token_ids) |
| 680 | if parser is not None and self._include_reasoning_tokens_details: |
| 681 | generated_token_ids[i].extend(output.token_ids) |
| 682 | if include_continuous_usage: |
| 683 | previous_reasoning_tokens[i] = ( |
| 684 | parser.count_reasoning_tokens( |
| 685 | tuple(generated_token_ids[i]) |
| 686 | ) |
| 687 | ) |
| 688 | |
| 689 | # if the message delta is None (e.g. because it was a |
| 690 | # "control token" for tool calls or the parser otherwise |
| 691 | # wasn't ready to send a token, then |
| 692 | # get the next token without streaming a chunk |
| 693 | # When reasoning is hidden, suppress per-token |
| 694 | # metadata (logprobs, token_ids) on every chunk to |
| 695 | # prevent leaking reasoning tokens through decoded |
| 696 | # token text in logprob entries or raw token IDs. |
| 697 | hide_stream_metadata = ( |
| 698 | not request.include_reasoning and parser is not None |
| 699 | ) |
| 700 | if hide_stream_metadata: |
| 701 | logprobs = None |
| 702 | |
| 703 | if delta_message is None: |
| 704 | # NOTE: If return_token_ids or logprobs are enabled, we |
| 705 | # still need to send a chunk even if delta_message is None |
| 706 | # to ensure all tokens are included in the response |
| 707 | if ( |
| 708 | output.finish_reason is None |
| 709 | and logprobs is None |
| 710 | and (not request.return_token_ids or hide_stream_metadata) |
| 711 | ): |
| 712 | continue |
| 713 | delta_message = DeltaMessage() |
| 714 | |
| 715 | # Log streaming delta if output logging is enabled |
| 716 | if self.enable_log_outputs and self.request_logger: |
| 717 | delta_content_parts = [] |
| 718 | if delta_message.content: |
| 719 | delta_content_parts.append(delta_message.content) |
| 720 | if delta_message.reasoning: |
| 721 | reasoning = delta_message.reasoning |
| 722 | delta_content_parts.append(f"[reasoning: {reasoning}]") |
| 723 | if delta_message.tool_calls: |
| 724 | tool_args = "".join( |
| 725 | tc.function.arguments |
| 726 | for tc in delta_message.tool_calls |
| 727 | if tc.function and tc.function.arguments |
| 728 | ) |
| 729 | if tool_args: |
| 730 | delta_content_parts.append(f"[tool_calls: {tool_args}]") |
| 731 | |
| 732 | if delta_content_parts and self.enable_log_deltas: |
| 733 | delta_content = " ".join(delta_content_parts) |
| 734 | self.request_logger.log_outputs( |
| 735 | request_id=request_id, |
| 736 | outputs=delta_content, |
| 737 | output_token_ids=as_list(output.token_ids), |
| 738 | finish_reason=output.finish_reason, |
| 739 | is_streaming=True, |
| 740 | delta=True, |
| 741 | ) |
| 742 | |
| 743 | include_token_ids = ( |
| 744 | request.return_token_ids and not hide_stream_metadata |
| 745 | ) |
| 746 | |
| 747 | if output.finish_reason is None: |
| 748 | # Send token-by-token response for each request.n |
| 749 | choice_data = ChatCompletionResponseStreamChoice( |
| 750 | index=i, |
| 751 | delta=delta_message, |
| 752 | logprobs=logprobs, |
| 753 | finish_reason=None, |
| 754 | token_ids=( |
| 755 | as_list(output.token_ids) if include_token_ids else None |
| 756 | ), |
| 757 | ) |
| 758 | |
| 759 | # if the model is finished generating |
| 760 | else: |
| 761 | # check for error finish reason and abort streaming |
| 762 | # finish_reason='error' indicates a retryable error |
| 763 | self._raise_if_error(output.finish_reason, request_id) |
| 764 | |
| 765 | # Send the finish response for each request.n only once |
| 766 | # In OpenAI's API, when a tool is called, the |
| 767 | # finish_reason is: |
| 768 | # "tool_calls" for "auto" or "required" tool calls, |
| 769 | # and "stop" for named tool calls. |
| 770 | if ( |
| 771 | tools_streamed[i] |
| 772 | and not tool_choice_function_name |
| 773 | and output.finish_reason == "stop" |
| 774 | ): |
| 775 | finish_reason_ = "tool_calls" |
| 776 | else: |
| 777 | finish_reason_ = ( |
| 778 | output.finish_reason if output.finish_reason else "stop" |
| 779 | ) |
| 780 | choice_data = ChatCompletionResponseStreamChoice( |
| 781 | index=i, |
| 782 | delta=delta_message, |
| 783 | logprobs=logprobs, |
| 784 | finish_reason=finish_reason_, |
| 785 | stop_reason=output.stop_reason, |
| 786 | token_ids=( |
| 787 | as_list(output.token_ids) if include_token_ids else None |
| 788 | ), |
| 789 | ) |
| 790 | |
| 791 | finish_reason_sent[i] = True |
| 792 | |
| 793 | choice_data = maybe_filter_parallel_tool_calls(choice_data, request) |
| 794 | chunk = ChatCompletionStreamResponse( |
| 795 | id=request_id, |
| 796 | object=chunk_object_type, |
| 797 | created=created_time, |
| 798 | choices=[choice_data], |
| 799 | model=model_name, |
| 800 | ) |
| 801 | # Stamp the fingerprint on terminal chunks only (those with |
| 802 | # finish_reason set). When ``include_usage`` is on, the |
| 803 | # trailing usage chunk below overrides this as the true |
| 804 | # final message. |
| 805 | if ( |
| 806 | not include_usage |
| 807 | and self.system_fingerprint is not None |
| 808 | and choice_data.finish_reason is not None |
| 809 | ): |
| 810 | chunk.system_fingerprint = self.system_fingerprint |
| 811 | |
| 812 | # handle usage stats if requested & if continuous |
| 813 | if include_continuous_usage: |
| 814 | completion_tokens = previous_num_tokens[i] |
| 815 | chunk.usage = UsageInfo( |
| 816 | prompt_tokens=num_prompt_tokens, |
| 817 | completion_tokens=completion_tokens, |
| 818 | total_tokens=num_prompt_tokens + completion_tokens, |
| 819 | completion_tokens_details=( |
| 820 | _make_completion_tokens_details( |
| 821 | previous_reasoning_tokens[i] |
| 822 | ) |
| 823 | if self._include_reasoning_tokens_details |
| 824 | else None |
| 825 | ), |
| 826 | ) |
| 827 | |
| 828 | data = chunk.model_dump_json(exclude_unset=True) |
| 829 | yield f"data: {data}\n\n" |
| 830 | |
| 831 | if self._include_reasoning_tokens_details and not include_continuous_usage: |
| 832 | for i, parser in enumerate(parsers): |
| 833 | if parser is not None and generated_token_ids[i]: |
| 834 | previous_reasoning_tokens[i] = parser.count_reasoning_tokens( |
| 835 | tuple(generated_token_ids[i]) |
| 836 | ) |
| 837 | |
| 838 | # once the final token is handled, if stream_options.include_usage |
| 839 | # is sent, send the usage |
| 840 | if include_usage: |
| 841 | completion_tokens = sum(previous_num_tokens) |
| 842 | final_usage = UsageInfo( |
| 843 | prompt_tokens=num_prompt_tokens, |
| 844 | completion_tokens=completion_tokens, |
| 845 | total_tokens=num_prompt_tokens + completion_tokens, |
| 846 | completion_tokens_details=_make_completion_tokens_details( |
| 847 | sum(previous_reasoning_tokens) |
| 848 | ) |
| 849 | if self._include_reasoning_tokens_details |
| 850 | else None, |
| 851 | ) |
| 852 | final_usage.prompt_tokens_details = _make_prompt_tokens_details( |
| 853 | self.enable_prompt_tokens_details, |
| 854 | num_cached_tokens, |
| 855 | num_cache_creation_tokens, |
| 856 | mm_token_counts, |
| 857 | ) |
| 858 | |
| 859 | # In streaming, metrics ride on this final usage chunk, which is |
| 860 | # only emitted when usage reporting is enabled (i.e. |
| 861 | # ``stream_options.include_usage=true`` or |
| 862 | # ``--enable-force-include-usage``). |
| 863 | stream_per_request_metrics: PerRequestMetrics | None = None |
| 864 | # See note in chat_completion_full_generator: suppress for n>1. |
| 865 | if (request.n or 1) == 1: |
| 866 | if self.enable_per_request_metrics: |
| 867 | last_metrics = ( |
| 868 | last_res.metrics if last_res is not None else None |
| 869 | ) |
| 870 | stream_per_request_metrics = build_per_request_timing_metrics( |
| 871 | last_metrics, completion_tokens |
| 872 | ) |
| 873 | spec_stats = build_spec_decoding_metrics(last_res) |
| 874 | if spec_stats is not None: |
| 875 | if stream_per_request_metrics is None: |
| 876 | stream_per_request_metrics = PerRequestMetrics() |
| 877 | stream_per_request_metrics.speculative_decoding = spec_stats |
| 878 | |
| 879 | final_usage_chunk = ChatCompletionStreamResponse( |
| 880 | id=request_id, |
| 881 | object=chunk_object_type, |
| 882 | created=created_time, |
| 883 | choices=[], |
| 884 | model=model_name, |
| 885 | usage=final_usage, |
| 886 | system_fingerprint=self.system_fingerprint, |
| 887 | metrics=stream_per_request_metrics, |
| 888 | ) |
| 889 | final_usage_data = final_usage_chunk.model_dump_json( |
| 890 | exclude_unset=True, exclude_none=True |
| 891 | ) |
| 892 | yield f"data: {final_usage_data}\n\n" |
| 893 | |
| 894 | # report to FastAPI middleware aggregate usage across all choices |
| 895 | num_completion_tokens = sum(previous_num_tokens) |
| 896 | request_metadata.final_usage_info = UsageInfo( |
| 897 | prompt_tokens=num_prompt_tokens, |
| 898 | completion_tokens=num_completion_tokens, |
| 899 | total_tokens=num_prompt_tokens + num_completion_tokens, |
| 900 | completion_tokens_details=_make_completion_tokens_details( |
| 901 | sum(previous_reasoning_tokens) |
| 902 | ) |
| 903 | if self._include_reasoning_tokens_details |
| 904 | else None, |
| 905 | ) |
| 906 | |
| 907 | # Log complete streaming response if output logging is enabled |
| 908 | if self.enable_log_outputs and self.request_logger: |
| 909 | # Log the complete response for each choice |
| 910 | for i in range(num_choices): |
| 911 | full_text = ( |
| 912 | previous_texts[i] |
| 913 | if previous_texts and i < len(previous_texts) |
| 914 | else f"<streaming_complete: {previous_num_tokens[i]} tokens>" |
| 915 | ) |
| 916 | self.request_logger.log_outputs( |
| 917 | request_id=request_id, |
| 918 | outputs=full_text, |
| 919 | output_token_ids=None, # Consider also logging all token IDs |
| 920 | finish_reason="streaming_complete", |
| 921 | is_streaming=True, |
| 922 | delta=False, |
| 923 | ) |
| 924 | |
| 925 | except GenerationError as e: |
| 926 | yield f"data: {self._convert_generation_error_to_streaming_response(e)}\n\n" |
| 927 | except Exception as e: |
| 928 | logger.exception("Error in chat completion stream generator.") |
| 929 | data = self.create_streaming_error_response(e) |
| 930 | yield f"data: {data}\n\n" |
| 931 | # Send the final done message after all response.n are finished |
| 932 | yield "data: [DONE]\n\n" |
| 933 | |
| 934 | async def chat_completion_full_generator( |
| 935 | self, |
| 936 | request: ChatCompletionRequest, |
| 937 | result_generator: AsyncIterator[RequestOutput], |
| 938 | request_id: str, |
| 939 | model_name: str, |
| 940 | conversation: list[ConversationMessage], |
| 941 | tokenizer: TokenizerLike, |
| 942 | request_metadata: RequestResponseMetadata, |
| 943 | parser: Parser | None = None, |
| 944 | mm_token_counts: dict[str, int] | None = None, |
| 945 | ) -> ErrorResponse | ChatCompletionResponse: |
| 946 | created_time = int(time.time()) |
| 947 | final_res: RequestOutput | None = None |
| 948 | |
| 949 | try: |
| 950 | async for res in result_generator: |
| 951 | final_res = res |
| 952 | except asyncio.CancelledError: |
| 953 | return self.create_error_response("Client disconnected") |
| 954 | |
| 955 | if final_res is None: |
| 956 | return self.create_error_response( |
| 957 | "No output received from the engine.", |
| 958 | err_type="InternalServerError", |
| 959 | status_code=HTTPStatus.INTERNAL_SERVER_ERROR, |
| 960 | ) |
| 961 | |
| 962 | choices: list[ChatCompletionResponseChoice] = [] |
| 963 | total_reasoning_tokens = 0 |
| 964 | |
| 965 | role = self.get_chat_request_role(request) |
| 966 | tool_parser_cls = ( |
| 967 | self.parser_cls.tool_parser_cls if self.parser_cls is not None else None |
| 968 | ) |
| 969 | for i, output in enumerate(final_res.outputs): |
| 970 | # check for error finish reason and raise GenerationError |
| 971 | # finish_reason='error' indicates a retryable request-level internal error |
| 972 | self._raise_if_error(output.finish_reason, request_id) |
| 973 | if i > 0 and parser is not None: |
| 974 | # Parsers are stateful: like streaming, use one per choice. |
| 975 | parser = self._make_parser( |
| 976 | request, tokenizer, self._effective_chat_template_kwargs(request) |
| 977 | ) |
| 978 | if parser is not None and final_res.prompt_token_ids is not None: |
| 979 | parser.set_prompt_token_ids(final_res.prompt_token_ids) |
| 980 | token_ids = output.token_ids |
| 981 | out_logprobs = output.logprobs |
| 982 | |
| 983 | if request.logprobs and ( |
| 984 | request.top_logprobs is not None or request.logprob_token_ids |
| 985 | ): |
| 986 | assert out_logprobs is not None, "Did not output logprobs" |
| 987 | logprobs = self._create_chat_logprobs( |
| 988 | token_ids=token_ids, |
| 989 | top_logprobs=out_logprobs, |
| 990 | num_output_top_logprobs=request.top_logprobs, |
| 991 | logprob_token_ids=request.logprob_token_ids, |
| 992 | tokenizer=tokenizer, |
| 993 | return_as_token_id=request.return_tokens_as_token_ids, |
| 994 | ) |
| 995 | else: |
| 996 | logprobs = None |
| 997 | |
| 998 | if parser is not None: |
| 999 | reasoning, content, tool_calls = parser.parse( |
| 1000 | output.text, |
| 1001 | request, |
| 1002 | enable_auto_tools=self.enable_auto_tools, |
| 1003 | model_output_token_ids=token_ids, |
| 1004 | ) |
| 1005 | suppress_metadata = not request.include_reasoning and parser is not None |
| 1006 | if not request.include_reasoning: |
| 1007 | reasoning = None |
| 1008 | total_reasoning_tokens += parser.count_reasoning_tokens(token_ids) |
| 1009 | if suppress_metadata: |
| 1010 | logprobs = None |
| 1011 | else: |
| 1012 | reasoning = None |
| 1013 | content = output.text |
| 1014 | tool_calls = [] |
| 1015 | suppress_metadata = False |
| 1016 | |
| 1017 | auto_tools_called = False |
| 1018 | is_named_tool_choice = ( |
| 1019 | request.tool_choice is not None |
| 1020 | and type(request.tool_choice) is ChatCompletionNamedToolChoiceParam |
| 1021 | ) |
| 1022 | is_required_tool_choice = request.tool_choice == "required" |
| 1023 | |
| 1024 | # All six construction sites route through ``self._create_chat_message`` |
| 1025 | # so subclasses can swap in a specialized :class:`ChatMessage` |
| 1026 | # (e.g. the Cohere v2 handler's ``CohereChatMessage``) without |
| 1027 | # having to duplicate this branch logic. |
| 1028 | if (not self.enable_auto_tools or not tool_parser_cls) and ( |
| 1029 | not is_named_tool_choice and not is_required_tool_choice |
| 1030 | ): |
| 1031 | message = self._create_chat_message( |
| 1032 | role=role, reasoning=reasoning, content=content |
| 1033 | ) |
| 1034 | |
| 1035 | elif is_named_tool_choice or is_required_tool_choice: |
| 1036 | message = self._create_chat_message( |
| 1037 | role=role, |
| 1038 | reasoning=reasoning, |
| 1039 | content=content or "", |
| 1040 | tool_calls=[ |
| 1041 | ToolCall(id=tc.id or make_tool_call_id(), function=tc) |
| 1042 | for tc in (tool_calls or []) |
| 1043 | ], |
| 1044 | ) |
| 1045 | |
| 1046 | # if the request doesn't use tool choice |
| 1047 | # OR specifies to not use a tool |
| 1048 | elif not request.tool_choice or request.tool_choice == "none": |
| 1049 | message = self._create_chat_message( |
| 1050 | role=role, reasoning=reasoning, content=content |
| 1051 | ) |
| 1052 | |
| 1053 | # handle when there are tools and tool choice is auto |
| 1054 | elif ( |
| 1055 | request.tools |
| 1056 | and (request.tool_choice == "auto" or request.tool_choice is None) |
| 1057 | and self.enable_auto_tools |
| 1058 | and tool_parser_cls |
| 1059 | ): |
| 1060 | auto_tools_called = tool_calls is not None and len(tool_calls) > 0 |
| 1061 | if tool_calls: |
| 1062 | message = self._create_chat_message( |
| 1063 | role=role, |
| 1064 | reasoning=reasoning, |
| 1065 | content=content, |
| 1066 | tool_calls=[ |
| 1067 | ToolCall(id=tc.id or make_tool_call_id(), function=tc) |
| 1068 | for tc in tool_calls |
| 1069 | ], |
| 1070 | ) |
| 1071 | |
| 1072 | else: |
| 1073 | message = self._create_chat_message( |
| 1074 | role=role, |
| 1075 | reasoning=reasoning, |
| 1076 | content=content, |
| 1077 | ) |
| 1078 | |
| 1079 | # undetermined case that is still important to handle |
| 1080 | else: |
| 1081 | logger.error( |
| 1082 | "Error in chat_completion_full_generator - cannot determine" |
| 1083 | " if tools should be extracted. Returning a standard chat " |
| 1084 | "completion." |
| 1085 | ) |
| 1086 | message = self._create_chat_message( |
| 1087 | role=role, reasoning=reasoning, content=content |
| 1088 | ) |
| 1089 | |
| 1090 | # Subclass hook: enrich the constructed message with any |
| 1091 | # parser-side extras that don't fit through the plain |
| 1092 | # ``(reasoning, content, tool_calls)`` tuple. Base is a no-op; |
| 1093 | # citation-aware handlers use this to surface grounding |
| 1094 | # metadata cached on the reasoning parser. |
| 1095 | message = self._finalize_response_message(message, parser=parser) |
| 1096 | |
| 1097 | # In OpenAI's API, when a tool is called, the finish_reason is: |
| 1098 | # "tool_calls" for "auto" or "required" tool calls, |
| 1099 | # and "stop" for named tool calls. |
| 1100 | is_finish_reason_tool_calls = auto_tools_called or ( |
| 1101 | request.tool_choice |
| 1102 | and request.tool_choice == "required" |
| 1103 | and output.finish_reason == "stop" |
| 1104 | ) |
| 1105 | |
| 1106 | routed_experts_b64 = ( |
| 1107 | numpy2base64(output.routed_experts) |
| 1108 | if output.routed_experts is not None |
| 1109 | else None |
| 1110 | ) |
| 1111 | |
| 1112 | choice_data = ChatCompletionResponseChoice( |
| 1113 | index=output.index, |
| 1114 | message=message, |
| 1115 | logprobs=logprobs, |
| 1116 | finish_reason="tool_calls" |
| 1117 | if is_finish_reason_tool_calls |
| 1118 | else output.finish_reason |
| 1119 | if output.finish_reason |
| 1120 | else "stop", |
| 1121 | stop_reason=output.stop_reason, |
| 1122 | token_ids=( |
| 1123 | as_list(output.token_ids) |
| 1124 | if request.return_token_ids and not suppress_metadata |
| 1125 | else None |
| 1126 | ), |
| 1127 | routed_experts=routed_experts_b64, |
| 1128 | ) |
| 1129 | choice_data = maybe_filter_parallel_tool_calls(choice_data, request) |
| 1130 | |
| 1131 | choices.append(choice_data) |
| 1132 | |
| 1133 | if request.echo: |
| 1134 | last_msg_content: str | list[dict[str, str]] = "" |
| 1135 | if ( |
| 1136 | conversation |
| 1137 | and "content" in conversation[-1] |
| 1138 | and conversation[-1].get("role") == role |
| 1139 | ): |
| 1140 | last_msg_content = conversation[-1]["content"] or "" |
| 1141 | if isinstance(last_msg_content, list): |
| 1142 | last_msg_content = "\n".join(msg["text"] for msg in last_msg_content) |
| 1143 | |
| 1144 | for choice in choices: |
| 1145 | full_message = last_msg_content + (choice.message.content or "") |
| 1146 | choice.message.content = full_message |
| 1147 | |
| 1148 | assert final_res.prompt_token_ids is not None |
| 1149 | num_prompt_tokens = len(final_res.prompt_token_ids) |
| 1150 | if final_res.encoder_prompt_token_ids is not None: |
| 1151 | num_prompt_tokens += len(final_res.encoder_prompt_token_ids) |
| 1152 | num_generated_tokens = sum( |
| 1153 | len(output.token_ids) for output in final_res.outputs |
| 1154 | ) |
| 1155 | usage = UsageInfo( |
| 1156 | prompt_tokens=num_prompt_tokens, |
| 1157 | completion_tokens=num_generated_tokens, |
| 1158 | total_tokens=num_prompt_tokens + num_generated_tokens, |
| 1159 | completion_tokens_details=_make_completion_tokens_details( |
| 1160 | total_reasoning_tokens |
| 1161 | ) |
| 1162 | if self._include_reasoning_tokens_details |
| 1163 | else None, |
| 1164 | ) |
| 1165 | usage.prompt_tokens_details = _make_prompt_tokens_details( |
| 1166 | self.enable_prompt_tokens_details, |
| 1167 | final_res.num_cached_tokens, |
| 1168 | final_res.num_cache_creation_tokens, |
| 1169 | mm_token_counts, |
| 1170 | ) |
| 1171 | |
| 1172 | request_metadata.final_usage_info = usage |
| 1173 | |
| 1174 | per_request_metrics: PerRequestMetrics | None = None |
| 1175 | # Per-request metrics (timing + spec-decode acceptance) describe a single |
| 1176 | # generation stream. For n>1 the stats belong to only one of the n |
| 1177 | # sequences, so they cannot be attributed to the request; suppress. |
| 1178 | if (request.n or 1) == 1: |
| 1179 | if self.enable_per_request_metrics: |
| 1180 | per_request_metrics = build_per_request_timing_metrics( |
| 1181 | final_res.metrics, num_generated_tokens |
| 1182 | ) |
| 1183 | spec_stats = build_spec_decoding_metrics(final_res) |
| 1184 | if spec_stats is not None: |
| 1185 | if per_request_metrics is None: |
| 1186 | per_request_metrics = PerRequestMetrics() |
| 1187 | per_request_metrics.speculative_decoding = spec_stats |
| 1188 | |
| 1189 | # ``final_res.prompt`` is the rendered chat-templated prompt text |
| 1190 | prompt_text = final_res.prompt if request.return_prompt_text else None |
| 1191 | |
| 1192 | response = ChatCompletionResponse( |
| 1193 | id=request_id, |
| 1194 | created=created_time, |
| 1195 | model=model_name, |
| 1196 | choices=choices, |
| 1197 | usage=usage, |
| 1198 | system_fingerprint=self.system_fingerprint, |
| 1199 | prompt_logprobs=clamp_prompt_logprobs(final_res.prompt_logprobs), |
| 1200 | prompt_token_ids=( |
| 1201 | final_res.prompt_token_ids if request.return_token_ids else None |
| 1202 | ), |
| 1203 | prompt_text=prompt_text, |
| 1204 | kv_transfer_params=final_res.kv_transfer_params, |
| 1205 | ec_transfer_params=final_res.ec_transfer_params, |
| 1206 | metrics=per_request_metrics, |
| 1207 | ) |
| 1208 | |
| 1209 | # Log complete response if output logging is enabled |
| 1210 | if self.enable_log_outputs and self.request_logger: |
| 1211 | for choice in choices: |
| 1212 | output_text = "" |
| 1213 | if choice.message.content: |
| 1214 | output_text = choice.message.content |
| 1215 | elif choice.message.tool_calls: |
| 1216 | # For tool calls, log the function name and arguments |
| 1217 | tool_call_descriptions = [] |
| 1218 | for tc in choice.message.tool_calls: # type: ignore |
| 1219 | function_call: FunctionCall = tc.function # type: ignore |
| 1220 | tool_call_descriptions.append( |
| 1221 | f"{function_call.name}({function_call.arguments})" |
| 1222 | ) |
| 1223 | tool_calls_str = ", ".join(tool_call_descriptions) |
| 1224 | output_text = f"[tool_calls: {tool_calls_str}]" |
| 1225 | |
| 1226 | if output_text: |
| 1227 | # Get the corresponding output token IDs |
| 1228 | output_token_ids = None |
| 1229 | if choice.index < len(final_res.outputs): |
| 1230 | output_token_ids = final_res.outputs[choice.index].token_ids |
| 1231 | |
| 1232 | self.request_logger.log_outputs( |
| 1233 | request_id=request_id, |
| 1234 | outputs=output_text, |
| 1235 | output_token_ids=output_token_ids, |
| 1236 | finish_reason=choice.finish_reason, |
| 1237 | is_streaming=False, |
| 1238 | delta=False, |
| 1239 | ) |
| 1240 | |
| 1241 | return response |
| 1242 | |
| 1243 | def _get_top_logprobs( |
| 1244 | self, |
| 1245 | logprobs: dict[int, Logprob], |
| 1246 | top_logprobs: int | None, |
| 1247 | tokenizer: TokenizerLike | None, |
| 1248 | should_return_as_token_id: bool, |
| 1249 | return_all: bool = False, |
| 1250 | ) -> list[ChatCompletionLogProb]: |
| 1251 | return [ |
| 1252 | ChatCompletionLogProb( |
| 1253 | token=( |
| 1254 | token := self._get_decoded_token( |
| 1255 | p[1], |
| 1256 | p[0], |
| 1257 | tokenizer, |
| 1258 | return_as_token_id=should_return_as_token_id, |
| 1259 | ) |
| 1260 | ), |
| 1261 | logprob=max(p[1].logprob, -9999.0), |
| 1262 | bytes=list(token.encode("utf-8", errors="replace")), |
| 1263 | ) |
| 1264 | for i, p in enumerate(logprobs.items()) |
| 1265 | if return_all |
| 1266 | or top_logprobs == -1 |
| 1267 | or (top_logprobs is not None and i < top_logprobs) |
| 1268 | ] |
| 1269 | |
| 1270 | def _create_chat_logprobs( |
| 1271 | self, |
| 1272 | token_ids: GenericSequence[int], |
| 1273 | top_logprobs: GenericSequence[dict[int, Logprob] | None], |
| 1274 | tokenizer: TokenizerLike | None, |
| 1275 | num_output_top_logprobs: int | None = None, |
| 1276 | logprob_token_ids: list[int] | None = None, |
| 1277 | return_as_token_id: bool | None = None, |
| 1278 | ) -> ChatCompletionLogProbs: |
| 1279 | """Create OpenAI-style logprobs.""" |
| 1280 | logprobs_content: list[ChatCompletionLogProbsContent] = [] |
| 1281 | |
| 1282 | should_return_as_token_id = ( |
| 1283 | return_as_token_id |
| 1284 | if return_as_token_id is not None |
| 1285 | else self.return_tokens_as_token_ids |
| 1286 | ) |
| 1287 | for i, token_id in enumerate(token_ids): |
| 1288 | step_top_logprobs = top_logprobs[i] |
| 1289 | if step_top_logprobs is None or step_top_logprobs.get(token_id) is None: |
| 1290 | if should_return_as_token_id: |
| 1291 | token = format_token_id_placeholder(token_id) |
| 1292 | else: |
| 1293 | if tokenizer is None: |
| 1294 | raise ValueError( |
| 1295 | "Unable to get tokenizer because `skip_tokenizer_init=True`" |
| 1296 | ) |
| 1297 | |
| 1298 | token = tokenizer.decode(token_id) |
| 1299 | |
| 1300 | logprobs_content.append( |
| 1301 | ChatCompletionLogProbsContent( |
| 1302 | token=token, |
| 1303 | bytes=list(token.encode("utf-8", errors="replace")), |
| 1304 | ) |
| 1305 | ) |
| 1306 | else: |
| 1307 | step_token = step_top_logprobs[token_id] |
| 1308 | step_decoded = step_token.decoded_token |
| 1309 | |
| 1310 | logprobs_content.append( |
| 1311 | ChatCompletionLogProbsContent( |
| 1312 | token=self._get_decoded_token( |
| 1313 | step_token, |
| 1314 | token_id, |
| 1315 | tokenizer, |
| 1316 | should_return_as_token_id, |
| 1317 | ), |
| 1318 | logprob=max(step_token.logprob, -9999.0), |
| 1319 | bytes=( |
| 1320 | None |
| 1321 | if step_decoded is None |
| 1322 | else list(step_decoded.encode("utf-8", errors="replace")) |
| 1323 | ), |
| 1324 | top_logprobs=self._get_top_logprobs( |
| 1325 | step_top_logprobs, |
| 1326 | num_output_top_logprobs, |
| 1327 | tokenizer, |
| 1328 | should_return_as_token_id, |
| 1329 | return_all=bool(logprob_token_ids), |
| 1330 | ), |
| 1331 | ) |
| 1332 | ) |
| 1333 | |
| 1334 | return ChatCompletionLogProbs(content=logprobs_content) |