| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import asyncio |
| 5 | import time |
| 6 | from collections.abc import AsyncGenerator, AsyncIterator |
| 7 | from collections.abc import Sequence as GenericSequence |
| 8 | from typing import cast |
| 9 | |
| 10 | from fastapi import Request |
| 11 | |
| 12 | from vllm.engine.protocol import EngineClient |
| 13 | from vllm.entrypoints.generate.base.protocol import ( |
| 14 | PerRequestMetrics, |
| 15 | RequestResponseMetadata, |
| 16 | ) |
| 17 | from vllm.entrypoints.generate.base.serving import ( |
| 18 | GenerateBaseServing, |
| 19 | build_per_request_timing_metrics, |
| 20 | build_spec_decoding_metrics, |
| 21 | clamp_prompt_logprobs, |
| 22 | format_token_id_placeholder, |
| 23 | ) |
| 24 | from vllm.entrypoints.openai.completion.protocol import ( |
| 25 | CompletionLogProbs, |
| 26 | CompletionRequest, |
| 27 | CompletionResponse, |
| 28 | CompletionResponseChoice, |
| 29 | CompletionResponseStreamChoice, |
| 30 | CompletionStreamResponse, |
| 31 | ) |
| 32 | from vllm.entrypoints.openai.models.serving import OpenAIServingModels |
| 33 | from vllm.entrypoints.serve.engine.protocol import ( |
| 34 | ErrorResponse, |
| 35 | PromptTokenUsageInfo, |
| 36 | UsageInfo, |
| 37 | ) |
| 38 | from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage |
| 39 | from vllm.entrypoints.serve.utils.request_logger import RequestLogger |
| 40 | from vllm.exceptions import GenerationError, VLLMValidationError |
| 41 | from vllm.inputs import EngineInput |
| 42 | from vllm.logger import init_logger |
| 43 | from vllm.logprobs import Logprob |
| 44 | from vllm.outputs import RequestOutput |
| 45 | from vllm.renderers.online_renderer import OnlineRenderer |
| 46 | from vllm.sampling_params import BeamSearchParams, SamplingParams |
| 47 | from vllm.tokenizers import TokenizerLike |
| 48 | from vllm.utils.async_utils import merge_async_iterators |
| 49 | from vllm.utils.collection_utils import as_list |
| 50 | from vllm.utils.serial_utils import numpy2base64 |
| 51 | |
| 52 | logger = init_logger(__name__) |
| 53 | |
| 54 | |
| 55 | class OpenAIServingCompletion(GenerateBaseServing): |
| 56 | def __init__( |
| 57 | self, |
| 58 | engine_client: EngineClient, |
| 59 | models: OpenAIServingModels, |
| 60 | *, |
| 61 | online_renderer: "OnlineRenderer", |
| 62 | request_logger: RequestLogger | None, |
| 63 | return_tokens_as_token_ids: bool = False, |
| 64 | enable_prompt_tokens_details: bool = False, |
| 65 | enable_force_include_usage: bool = False, |
| 66 | enable_per_request_metrics: bool = False, |
| 67 | ): |
| 68 | super().__init__( |
| 69 | engine_client=engine_client, |
| 70 | models=models, |
| 71 | request_logger=request_logger, |
| 72 | return_tokens_as_token_ids=return_tokens_as_token_ids, |
| 73 | ) |
| 74 | |
| 75 | self.online_renderer = online_renderer |
| 76 | self.enable_prompt_tokens_details = enable_prompt_tokens_details |
| 77 | self.enable_force_include_usage = enable_force_include_usage |
| 78 | self.enable_per_request_metrics = enable_per_request_metrics |
| 79 | |
| 80 | self.default_sampling_params = self.model_config.get_diff_sampling_param() |
| 81 | mc = self.model_config |
| 82 | self.override_max_tokens = ( |
| 83 | self.default_sampling_params.get("max_tokens") |
| 84 | if mc.generation_config not in ("auto", "vllm") |
| 85 | else getattr(mc, "override_generation_config", {}).get("max_new_tokens") |
| 86 | ) |
| 87 | |
| 88 | async def render_completion_request( |
| 89 | self, |
| 90 | request: CompletionRequest, |
| 91 | ) -> list[EngineInput] | ErrorResponse: |
| 92 | """Validate the model and preprocess a completion request. |
| 93 | |
| 94 | Delegates preprocessing logic to OnlineRenderer, adding the |
| 95 | engine-aware checks (LoRA model validation, engine health). |
| 96 | |
| 97 | Returns: |
| 98 | A list of engine_inputs on success, or an ErrorResponse on failure. |
| 99 | |
| 100 | """ |
| 101 | error_check_ret = await self._check_model(request) |
| 102 | if error_check_ret is not None: |
| 103 | return error_check_ret |
| 104 | |
| 105 | self._preflight(request.n or 1) |
| 106 | |
| 107 | return await self.online_renderer.render_completion(request) |
| 108 | |
| 109 | async def create_completion( |
| 110 | self, |
| 111 | request: CompletionRequest, |
| 112 | raw_request: Request | None = None, |
| 113 | ) -> AsyncGenerator[str, None] | CompletionResponse | ErrorResponse: |
| 114 | """Completion API similar to OpenAI's API. |
| 115 | |
| 116 | See https://platform.openai.com/docs/api-reference/completions/create |
| 117 | for the API specification. This API mimics the OpenAI Completion API. |
| 118 | |
| 119 | NOTE: suffix is only supported by models that implement FIM rendering. |
| 120 | """ |
| 121 | return await self._with_kv_transfer_rejection_cleanup( |
| 122 | self._create_completion(request, raw_request), request, raw_request |
| 123 | ) |
| 124 | |
| 125 | async def _create_completion( |
| 126 | self, |
| 127 | request: CompletionRequest, |
| 128 | raw_request: Request | None = None, |
| 129 | ) -> AsyncGenerator[str, None] | CompletionResponse | ErrorResponse: |
| 130 | if request.stream and request.use_beam_search: |
| 131 | return self.create_error_response( |
| 132 | "Streaming is not currently supported with beam search" |
| 133 | ) |
| 134 | |
| 135 | result = await self.render_completion_request(request) |
| 136 | if isinstance(result, ErrorResponse): |
| 137 | return result |
| 138 | |
| 139 | engine_inputs = result |
| 140 | |
| 141 | request_id = f"cmpl-{self._base_request_id(raw_request, request.request_id)}" |
| 142 | created_time = int(time.time()) |
| 143 | |
| 144 | request_metadata = RequestResponseMetadata(request_id=request_id) |
| 145 | if raw_request: |
| 146 | raw_request.state.request_metadata = request_metadata |
| 147 | |
| 148 | lora_request = self._maybe_get_adapters(request) |
| 149 | |
| 150 | # Extract data_parallel_rank from header (router can inject it) |
| 151 | data_parallel_rank = self._get_data_parallel_rank(raw_request) |
| 152 | |
| 153 | # Schedule the request and get the result generator. |
| 154 | max_model_len = self.model_config.max_model_len |
| 155 | generators: list[AsyncGenerator[RequestOutput, None]] = [] |
| 156 | for i, engine_input in enumerate(engine_inputs): |
| 157 | max_tokens = get_max_tokens( |
| 158 | max_model_len, |
| 159 | request.max_tokens, |
| 160 | self._extract_prompt_len(engine_input), |
| 161 | self.default_sampling_params, |
| 162 | self.override_max_tokens, |
| 163 | truncate_prompt_tokens=request.truncate_prompt_tokens, |
| 164 | ) |
| 165 | |
| 166 | sampling_params: SamplingParams | BeamSearchParams |
| 167 | if request.use_beam_search: |
| 168 | sampling_params = request.to_beam_search_params( |
| 169 | max_tokens, self.default_sampling_params |
| 170 | ) |
| 171 | else: |
| 172 | sampling_params = request.to_sampling_params( |
| 173 | max_tokens, |
| 174 | self.default_sampling_params, |
| 175 | ) |
| 176 | |
| 177 | request_id_item = f"{request_id}-{i}" |
| 178 | |
| 179 | self._log_inputs( |
| 180 | request_id_item, |
| 181 | engine_input, |
| 182 | params=sampling_params, |
| 183 | lora_request=lora_request, |
| 184 | ) |
| 185 | |
| 186 | trace_headers = ( |
| 187 | None |
| 188 | if raw_request is None |
| 189 | else await self._get_trace_headers(raw_request.headers) |
| 190 | ) |
| 191 | session_id = self._get_session_id(request, raw_request) |
| 192 | |
| 193 | if isinstance(sampling_params, BeamSearchParams): |
| 194 | generator = self.beam_search( |
| 195 | prompt=engine_input, |
| 196 | request_id=request_id, |
| 197 | params=sampling_params, |
| 198 | lora_request=lora_request, |
| 199 | trace_headers=trace_headers, |
| 200 | session_id=session_id, |
| 201 | ) |
| 202 | else: |
| 203 | generator = self.engine_client.generate( |
| 204 | engine_input, |
| 205 | sampling_params, |
| 206 | request_id_item, |
| 207 | lora_request=lora_request, |
| 208 | trace_headers=trace_headers, |
| 209 | priority=self._get_priority(request, raw_request), |
| 210 | data_parallel_rank=data_parallel_rank, |
| 211 | session_id=session_id, |
| 212 | ) |
| 213 | |
| 214 | generators.append(generator) |
| 215 | |
| 216 | result_generator = merge_async_iterators(*generators) |
| 217 | |
| 218 | model_name = self.models.model_name(lora_request) |
| 219 | num_prompts = len(engine_inputs) |
| 220 | |
| 221 | # Streaming response |
| 222 | tokenizer = self.renderer.tokenizer |
| 223 | |
| 224 | if request.stream: |
| 225 | return self.completion_stream_generator( |
| 226 | request, |
| 227 | engine_inputs, |
| 228 | result_generator, |
| 229 | request_id, |
| 230 | created_time, |
| 231 | model_name, |
| 232 | num_prompts=num_prompts, |
| 233 | tokenizer=tokenizer, |
| 234 | request_metadata=request_metadata, |
| 235 | ) |
| 236 | |
| 237 | # Non-streaming response |
| 238 | final_res_batch: list[RequestOutput | None] = [None] * num_prompts |
| 239 | try: |
| 240 | async for i, res in result_generator: |
| 241 | final_res_batch[i] = res |
| 242 | |
| 243 | for i, final_res in enumerate(final_res_batch): |
| 244 | assert final_res is not None |
| 245 | |
| 246 | # The output should contain the input text |
| 247 | # We did not pass it into vLLM engine to avoid being redundant |
| 248 | # with the inputs token IDs |
| 249 | if final_res.prompt is None: |
| 250 | final_res.prompt = self._extract_prompt_text(engine_inputs[i]) |
| 251 | |
| 252 | final_res_batch_checked = cast(list[RequestOutput], final_res_batch) |
| 253 | |
| 254 | response = self.request_output_to_completion_response( |
| 255 | final_res_batch_checked, |
| 256 | request, |
| 257 | request_id, |
| 258 | created_time, |
| 259 | model_name, |
| 260 | tokenizer, |
| 261 | request_metadata, |
| 262 | ) |
| 263 | except asyncio.CancelledError: |
| 264 | return self.create_error_response("Client disconnected") |
| 265 | |
| 266 | # When user requests streaming but we don't stream, we still need to |
| 267 | # return a streaming response with a single event. |
| 268 | if request.stream: |
| 269 | response_json = response.model_dump_json() |
| 270 | |
| 271 | async def fake_stream_generator() -> AsyncGenerator[str, None]: |
| 272 | yield f"data: {response_json}\n\n" |
| 273 | yield "data: [DONE]\n\n" |
| 274 | |
| 275 | return fake_stream_generator() |
| 276 | |
| 277 | return response |
| 278 | |
| 279 | async def completion_stream_generator( |
| 280 | self, |
| 281 | request: CompletionRequest, |
| 282 | engine_inputs: list[EngineInput], |
| 283 | result_generator: AsyncIterator[tuple[int, RequestOutput]], |
| 284 | request_id: str, |
| 285 | created_time: int, |
| 286 | model_name: str, |
| 287 | num_prompts: int, |
| 288 | tokenizer: TokenizerLike | None, |
| 289 | request_metadata: RequestResponseMetadata, |
| 290 | ) -> AsyncGenerator[str, None]: |
| 291 | num_choices = 1 if request.n is None else request.n |
| 292 | previous_text_lens = [0] * num_choices * num_prompts |
| 293 | previous_num_tokens = [0] * num_choices * num_prompts |
| 294 | has_echoed = [False] * num_choices * num_prompts |
| 295 | num_prompt_tokens = [0] * num_prompts |
| 296 | num_cached_tokens = None |
| 297 | first_iteration = True |
| 298 | |
| 299 | stream_options = request.stream_options |
| 300 | include_usage, include_continuous_usage = should_include_usage( |
| 301 | stream_options, self.enable_force_include_usage |
| 302 | ) |
| 303 | |
| 304 | last_res: RequestOutput | None = None |
| 305 | try: |
| 306 | async for prompt_idx, res in result_generator: |
| 307 | last_res = res |
| 308 | prompt_token_ids = res.prompt_token_ids |
| 309 | prompt_logprobs = res.prompt_logprobs |
| 310 | |
| 311 | if first_iteration: |
| 312 | num_cached_tokens = res.num_cached_tokens |
| 313 | first_iteration = False |
| 314 | |
| 315 | prompt_text = res.prompt |
| 316 | if prompt_text is None: |
| 317 | engine_input = engine_inputs[prompt_idx] |
| 318 | prompt_text = self._extract_prompt_text(engine_input) |
| 319 | |
| 320 | # Prompt details are excluded from later streamed outputs |
| 321 | if prompt_token_ids is not None: |
| 322 | num_prompt_tokens[prompt_idx] = len(prompt_token_ids) |
| 323 | |
| 324 | delta_token_ids: GenericSequence[int] |
| 325 | out_logprobs: GenericSequence[dict[int, Logprob] | None] | None |
| 326 | |
| 327 | for output in res.outputs: |
| 328 | i = output.index + prompt_idx * num_choices |
| 329 | |
| 330 | # Useful when request.return_token_ids is True |
| 331 | # Returning prompt token IDs shares the same logic |
| 332 | # with the echo implementation. |
| 333 | prompt_token_ids_to_return: list[int] | None = None |
| 334 | |
| 335 | assert request.max_tokens is not None |
| 336 | if request.echo and not has_echoed[i]: |
| 337 | assert prompt_token_ids is not None |
| 338 | if request.return_token_ids: |
| 339 | prompt_text = "" |
| 340 | assert prompt_text is not None |
| 341 | if request.max_tokens == 0: |
| 342 | # only return the prompt |
| 343 | delta_text = prompt_text |
| 344 | delta_token_ids = prompt_token_ids |
| 345 | out_logprobs = prompt_logprobs |
| 346 | else: |
| 347 | # echo the prompt and first token |
| 348 | delta_text = prompt_text + output.text |
| 349 | delta_token_ids = [ |
| 350 | *prompt_token_ids, |
| 351 | *output.token_ids, |
| 352 | ] |
| 353 | out_logprobs = [ |
| 354 | *(prompt_logprobs or []), |
| 355 | *(output.logprobs or []), |
| 356 | ] |
| 357 | prompt_token_ids_to_return = prompt_token_ids |
| 358 | has_echoed[i] = True |
| 359 | else: |
| 360 | # return just the delta |
| 361 | delta_text = output.text |
| 362 | delta_token_ids = output.token_ids |
| 363 | out_logprobs = output.logprobs |
| 364 | |
| 365 | # has_echoed[i] is reused here to indicate whether |
| 366 | # we have already returned the prompt token IDs. |
| 367 | if not has_echoed[i] and request.return_token_ids: |
| 368 | prompt_token_ids_to_return = prompt_token_ids |
| 369 | has_echoed[i] = True |
| 370 | |
| 371 | if ( |
| 372 | not delta_text |
| 373 | and not delta_token_ids |
| 374 | and not previous_num_tokens[i] |
| 375 | ): |
| 376 | # Chunked prefill case, don't return empty chunks |
| 377 | continue |
| 378 | |
| 379 | if request.logprobs is not None: |
| 380 | assert out_logprobs is not None, "Did not output logprobs" |
| 381 | logprobs = self._create_completion_logprobs( |
| 382 | token_ids=delta_token_ids, |
| 383 | top_logprobs=out_logprobs, |
| 384 | num_output_top_logprobs=request.logprobs, |
| 385 | tokenizer=tokenizer, |
| 386 | logprob_token_ids=request.logprob_token_ids, |
| 387 | initial_text_offset=previous_text_lens[i], |
| 388 | return_as_token_id=request.return_tokens_as_token_ids, |
| 389 | ) |
| 390 | else: |
| 391 | logprobs = None |
| 392 | |
| 393 | previous_text_lens[i] += len(delta_text) |
| 394 | previous_num_tokens[i] += len(output.token_ids) |
| 395 | finish_reason = output.finish_reason |
| 396 | stop_reason = output.stop_reason |
| 397 | |
| 398 | self._raise_if_error(finish_reason, request_id) |
| 399 | |
| 400 | chunk = CompletionStreamResponse( |
| 401 | id=request_id, |
| 402 | object="text_completion", |
| 403 | created=created_time, |
| 404 | model=model_name, |
| 405 | choices=[ |
| 406 | CompletionResponseStreamChoice( |
| 407 | index=i, |
| 408 | text=delta_text, |
| 409 | logprobs=logprobs, |
| 410 | finish_reason=finish_reason, |
| 411 | stop_reason=stop_reason, |
| 412 | prompt_token_ids=prompt_token_ids_to_return, |
| 413 | token_ids=( |
| 414 | as_list(output.token_ids) |
| 415 | if request.return_token_ids |
| 416 | else None |
| 417 | ), |
| 418 | ) |
| 419 | ], |
| 420 | ) |
| 421 | # Stamp on terminal chunk only when no trailing usage chunk |
| 422 | # will follow (that one is the true final message). |
| 423 | if ( |
| 424 | not include_usage |
| 425 | and self.system_fingerprint is not None |
| 426 | and finish_reason is not None |
| 427 | ): |
| 428 | chunk.system_fingerprint = self.system_fingerprint |
| 429 | if include_continuous_usage: |
| 430 | prompt_tokens = num_prompt_tokens[prompt_idx] |
| 431 | completion_tokens = previous_num_tokens[i] |
| 432 | chunk.usage = UsageInfo( |
| 433 | prompt_tokens=prompt_tokens, |
| 434 | completion_tokens=completion_tokens, |
| 435 | total_tokens=prompt_tokens + completion_tokens, |
| 436 | ) |
| 437 | |
| 438 | response_json = chunk.model_dump_json(exclude_unset=True) |
| 439 | yield f"data: {response_json}\n\n" |
| 440 | |
| 441 | total_prompt_tokens = sum(num_prompt_tokens) |
| 442 | total_completion_tokens = sum(previous_num_tokens) |
| 443 | final_usage_info = UsageInfo( |
| 444 | prompt_tokens=total_prompt_tokens, |
| 445 | completion_tokens=total_completion_tokens, |
| 446 | total_tokens=total_prompt_tokens + total_completion_tokens, |
| 447 | ) |
| 448 | |
| 449 | if self.enable_prompt_tokens_details and num_cached_tokens is not None: |
| 450 | final_usage_info.prompt_tokens_details = PromptTokenUsageInfo( |
| 451 | cached_tokens=num_cached_tokens |
| 452 | ) |
| 453 | |
| 454 | if include_usage: |
| 455 | # In streaming, metrics ride on this final usage chunk, which is |
| 456 | # only emitted when usage reporting is enabled (i.e. |
| 457 | # ``stream_options.include_usage=true`` or |
| 458 | # ``--enable-force-include-usage``). |
| 459 | stream_per_request_metrics: PerRequestMetrics | None = None |
| 460 | # See note in request_output_to_completion_response: suppress when |
| 461 | # not attributable to one stream (multi-prompt or n>1). |
| 462 | if num_prompts == 1 and (request.n or 1) == 1: |
| 463 | if self.enable_per_request_metrics: |
| 464 | last_metrics = ( |
| 465 | last_res.metrics if last_res is not None else None |
| 466 | ) |
| 467 | stream_per_request_metrics = build_per_request_timing_metrics( |
| 468 | last_metrics, total_completion_tokens |
| 469 | ) |
| 470 | spec_stats = build_spec_decoding_metrics(last_res) |
| 471 | if spec_stats is not None: |
| 472 | if stream_per_request_metrics is None: |
| 473 | stream_per_request_metrics = PerRequestMetrics() |
| 474 | stream_per_request_metrics.speculative_decoding = spec_stats |
| 475 | |
| 476 | final_usage_chunk = CompletionStreamResponse( |
| 477 | id=request_id, |
| 478 | created=created_time, |
| 479 | model=model_name, |
| 480 | choices=[], |
| 481 | usage=final_usage_info, |
| 482 | system_fingerprint=self.system_fingerprint, |
| 483 | metrics=stream_per_request_metrics, |
| 484 | ) |
| 485 | final_usage_data = final_usage_chunk.model_dump_json( |
| 486 | exclude_unset=False, exclude_none=True |
| 487 | ) |
| 488 | yield f"data: {final_usage_data}\n\n" |
| 489 | |
| 490 | # report to FastAPI middleware aggregate usage across all choices |
| 491 | request_metadata.final_usage_info = final_usage_info |
| 492 | |
| 493 | except GenerationError as e: |
| 494 | yield f"data: {self._convert_generation_error_to_streaming_response(e)}\n\n" |
| 495 | except Exception as e: |
| 496 | logger.exception("Error in completion stream generator.") |
| 497 | data = self.create_streaming_error_response(e) |
| 498 | yield f"data: {data}\n\n" |
| 499 | yield "data: [DONE]\n\n" |
| 500 | |
| 501 | def request_output_to_completion_response( |
| 502 | self, |
| 503 | final_res_batch: list[RequestOutput], |
| 504 | request: CompletionRequest, |
| 505 | request_id: str, |
| 506 | created_time: int, |
| 507 | model_name: str, |
| 508 | tokenizer: TokenizerLike | None, |
| 509 | request_metadata: RequestResponseMetadata, |
| 510 | ) -> CompletionResponse: |
| 511 | choices: list[CompletionResponseChoice] = [] |
| 512 | num_prompt_tokens = 0 |
| 513 | num_generated_tokens = 0 |
| 514 | kv_transfer_params = None |
| 515 | ec_transfer_params = None |
| 516 | last_final_res = None |
| 517 | for final_res in final_res_batch: |
| 518 | last_final_res = final_res |
| 519 | prompt_token_ids = final_res.prompt_token_ids |
| 520 | assert prompt_token_ids is not None |
| 521 | prompt_logprobs = clamp_prompt_logprobs(final_res.prompt_logprobs) |
| 522 | prompt_text = final_res.prompt |
| 523 | |
| 524 | token_ids: GenericSequence[int] |
| 525 | out_logprobs: GenericSequence[dict[int, Logprob] | None] | None |
| 526 | |
| 527 | for output in final_res.outputs: |
| 528 | self._raise_if_error(output.finish_reason, request_id) |
| 529 | |
| 530 | assert request.max_tokens is not None |
| 531 | if request.echo: |
| 532 | if request.return_token_ids: |
| 533 | prompt_text = "" |
| 534 | assert prompt_text is not None |
| 535 | if request.max_tokens == 0: |
| 536 | token_ids = prompt_token_ids |
| 537 | out_logprobs = prompt_logprobs |
| 538 | output_text = prompt_text |
| 539 | else: |
| 540 | token_ids = [*prompt_token_ids, *output.token_ids] |
| 541 | |
| 542 | if request.logprobs is None: |
| 543 | out_logprobs = None |
| 544 | else: |
| 545 | assert prompt_logprobs is not None |
| 546 | assert output.logprobs is not None |
| 547 | out_logprobs = [ |
| 548 | *prompt_logprobs, |
| 549 | *output.logprobs, |
| 550 | ] |
| 551 | |
| 552 | output_text = prompt_text + output.text |
| 553 | else: |
| 554 | token_ids = output.token_ids |
| 555 | out_logprobs = output.logprobs |
| 556 | output_text = output.text |
| 557 | |
| 558 | if request.logprobs is not None: |
| 559 | assert out_logprobs is not None, "Did not output logprobs" |
| 560 | logprobs = self._create_completion_logprobs( |
| 561 | token_ids=token_ids, |
| 562 | top_logprobs=out_logprobs, |
| 563 | tokenizer=tokenizer, |
| 564 | num_output_top_logprobs=request.logprobs, |
| 565 | logprob_token_ids=request.logprob_token_ids, |
| 566 | return_as_token_id=request.return_tokens_as_token_ids, |
| 567 | ) |
| 568 | else: |
| 569 | logprobs = None |
| 570 | |
| 571 | routed_experts_b64 = ( |
| 572 | numpy2base64(output.routed_experts) |
| 573 | if output.routed_experts is not None |
| 574 | else None |
| 575 | ) |
| 576 | |
| 577 | choice_data = CompletionResponseChoice( |
| 578 | index=len(choices), |
| 579 | text=output_text, |
| 580 | logprobs=logprobs, |
| 581 | finish_reason=output.finish_reason, |
| 582 | stop_reason=output.stop_reason, |
| 583 | prompt_logprobs=final_res.prompt_logprobs, |
| 584 | prompt_token_ids=( |
| 585 | prompt_token_ids if request.return_token_ids else None |
| 586 | ), |
| 587 | token_ids=( |
| 588 | as_list(output.token_ids) if request.return_token_ids else None |
| 589 | ), |
| 590 | routed_experts=routed_experts_b64, |
| 591 | ) |
| 592 | choices.append(choice_data) |
| 593 | |
| 594 | num_generated_tokens += len(output.token_ids) |
| 595 | |
| 596 | num_prompt_tokens += len(prompt_token_ids) |
| 597 | |
| 598 | usage = UsageInfo( |
| 599 | prompt_tokens=num_prompt_tokens, |
| 600 | completion_tokens=num_generated_tokens, |
| 601 | total_tokens=num_prompt_tokens + num_generated_tokens, |
| 602 | ) |
| 603 | |
| 604 | if ( |
| 605 | self.enable_prompt_tokens_details |
| 606 | and last_final_res |
| 607 | and last_final_res.num_cached_tokens is not None |
| 608 | ): |
| 609 | usage.prompt_tokens_details = PromptTokenUsageInfo( |
| 610 | cached_tokens=last_final_res.num_cached_tokens |
| 611 | ) |
| 612 | |
| 613 | request_metadata.final_usage_info = usage |
| 614 | |
| 615 | per_request_metrics: PerRequestMetrics | None = None |
| 616 | # Per-request metrics (timing + spec-decode acceptance) describe a single |
| 617 | # generation stream, so suppress them when they cannot be attributed to |
| 618 | # one: multiple prompts (timestamps span prompts) or n>1 (stats belong to |
| 619 | # one of the n sequences). |
| 620 | if len(final_res_batch) == 1 and (request.n or 1) == 1: |
| 621 | if self.enable_per_request_metrics: |
| 622 | last_metrics = ( |
| 623 | last_final_res.metrics if last_final_res is not None else None |
| 624 | ) |
| 625 | per_request_metrics = build_per_request_timing_metrics( |
| 626 | last_metrics, num_generated_tokens |
| 627 | ) |
| 628 | spec_stats = build_spec_decoding_metrics(last_final_res) |
| 629 | if spec_stats is not None: |
| 630 | if per_request_metrics is None: |
| 631 | per_request_metrics = PerRequestMetrics() |
| 632 | per_request_metrics.speculative_decoding = spec_stats |
| 633 | |
| 634 | if final_res_batch: |
| 635 | kv_transfer_params = final_res_batch[0].kv_transfer_params |
| 636 | ec_transfer_params = final_res_batch[0].ec_transfer_params |
| 637 | |
| 638 | return CompletionResponse( |
| 639 | id=request_id, |
| 640 | created=created_time, |
| 641 | model=model_name, |
| 642 | choices=choices, |
| 643 | usage=usage, |
| 644 | system_fingerprint=self.system_fingerprint, |
| 645 | kv_transfer_params=kv_transfer_params, |
| 646 | ec_transfer_params=ec_transfer_params, |
| 647 | metrics=per_request_metrics, |
| 648 | ) |
| 649 | |
| 650 | def _create_completion_logprobs( |
| 651 | self, |
| 652 | token_ids: GenericSequence[int], |
| 653 | top_logprobs: GenericSequence[dict[int, Logprob] | None], |
| 654 | num_output_top_logprobs: int, |
| 655 | tokenizer: TokenizerLike | None, |
| 656 | logprob_token_ids: list[int] | None = None, |
| 657 | initial_text_offset: int = 0, |
| 658 | return_as_token_id: bool | None = None, |
| 659 | ) -> CompletionLogProbs: |
| 660 | """Create logprobs for OpenAI Completion API.""" |
| 661 | out_text_offset: list[int] = [] |
| 662 | out_token_logprobs: list[float | None] = [] |
| 663 | out_tokens: list[str] = [] |
| 664 | out_top_logprobs: list[dict[str, float] | None] = [] |
| 665 | |
| 666 | last_token_len = 0 |
| 667 | |
| 668 | should_return_as_token_id = ( |
| 669 | return_as_token_id |
| 670 | if return_as_token_id is not None |
| 671 | else self.return_tokens_as_token_ids |
| 672 | ) |
| 673 | for i, token_id in enumerate(token_ids): |
| 674 | step_top_logprobs = top_logprobs[i] |
| 675 | if step_top_logprobs is None: |
| 676 | if should_return_as_token_id: |
| 677 | token = format_token_id_placeholder(token_id) |
| 678 | else: |
| 679 | if tokenizer is None: |
| 680 | raise VLLMValidationError( |
| 681 | "Unable to get tokenizer because " |
| 682 | "`skip_tokenizer_init=True`", |
| 683 | parameter="skip_tokenizer_init", |
| 684 | value=True, |
| 685 | ) |
| 686 | |
| 687 | token = tokenizer.decode(token_id) |
| 688 | |
| 689 | out_tokens.append(token) |
| 690 | out_token_logprobs.append(None) |
| 691 | out_top_logprobs.append(None) |
| 692 | else: |
| 693 | step_token = step_top_logprobs[token_id] |
| 694 | |
| 695 | token = self._get_decoded_token( |
| 696 | step_token, |
| 697 | token_id, |
| 698 | tokenizer, |
| 699 | return_as_token_id=should_return_as_token_id, |
| 700 | ) |
| 701 | token_logprob = max(step_token.logprob, -9999.0) |
| 702 | |
| 703 | out_tokens.append(token) |
| 704 | out_token_logprobs.append(token_logprob) |
| 705 | |
| 706 | # makes sure to add the top num_output_top_logprobs + 1 |
| 707 | # logprobs, as defined in the openai API |
| 708 | # (cf. https://github.com/openai/openai-openapi/blob/ |
| 709 | # 893ba52242dbd5387a97b96444ee1c742cfce9bd/openapi.yaml#L7153) |
| 710 | out_top_logprobs.append( |
| 711 | { |
| 712 | # Convert float("-inf") to the |
| 713 | # JSON-serializable float that OpenAI uses |
| 714 | self._get_decoded_token( |
| 715 | top_lp[1], |
| 716 | top_lp[0], |
| 717 | tokenizer, |
| 718 | return_as_token_id=should_return_as_token_id, |
| 719 | ): max(top_lp[1].logprob, -9999.0) |
| 720 | for i, top_lp in enumerate(step_top_logprobs.items()) |
| 721 | if logprob_token_ids or num_output_top_logprobs >= i |
| 722 | } |
| 723 | ) |
| 724 | |
| 725 | if len(out_text_offset) == 0: |
| 726 | out_text_offset.append(initial_text_offset) |
| 727 | else: |
| 728 | out_text_offset.append(out_text_offset[-1] + last_token_len) |
| 729 | last_token_len = len(token) |
| 730 | |
| 731 | return CompletionLogProbs( |
| 732 | text_offset=out_text_offset, |
| 733 | token_logprobs=out_token_logprobs, |
| 734 | tokens=out_tokens, |
| 735 | top_logprobs=out_top_logprobs, |
| 736 | ) |