| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | from collections.abc import Sequence |
| 4 | from typing import Any |
| 5 | |
| 6 | from vllm.config import ModelConfig |
| 7 | from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption |
| 8 | from vllm.entrypoints.generate.base.protocol import ( |
| 9 | DeltaMessage, |
| 10 | ToolCall, |
| 11 | ) |
| 12 | from vllm.entrypoints.generate.base.serving import resolve_token_id_placeholder |
| 13 | from vllm.entrypoints.openai.chat_completion.protocol import ( |
| 14 | ChatCompletionLogProbs, |
| 15 | ChatCompletionNamedToolChoiceParam, |
| 16 | ChatCompletionRequest, |
| 17 | ChatCompletionResponseChoice, |
| 18 | ChatCompletionResponseStreamChoice, |
| 19 | ChatCompletionStreamResponse, |
| 20 | ChatMessage, |
| 21 | ) |
| 22 | from vllm.entrypoints.openai.completion.protocol import ( |
| 23 | CompletionLogProbs, |
| 24 | CompletionRequest, |
| 25 | CompletionResponseChoice, |
| 26 | CompletionResponseStreamChoice, |
| 27 | CompletionStreamResponse, |
| 28 | ) |
| 29 | from vllm.entrypoints.scale_out.token_in_token_out.protocol import ( |
| 30 | DerenderStreamState, |
| 31 | GenerateResponse, |
| 32 | GenerateStreamResponse, |
| 33 | ) |
| 34 | from vllm.entrypoints.serve.engine.protocol import UsageInfo |
| 35 | from vllm.entrypoints.serve.utils.request_logger import RequestLogger |
| 36 | from vllm.entrypoints.serve.utils.tool_calls_utils import ( |
| 37 | maybe_filter_parallel_tool_calls, |
| 38 | ) |
| 39 | from vllm.logger import init_logger |
| 40 | from vllm.parser import Parser, ParserManager |
| 41 | from vllm.renderers import BaseRenderer |
| 42 | from vllm.tokenizers import TokenizerLike |
| 43 | from vllm.tokenizers.detokenizer_utils import ( |
| 44 | convert_prompt_ids_to_tokens, |
| 45 | detokenize_incrementally, |
| 46 | get_leading_space_marker, |
| 47 | ) |
| 48 | from vllm.utils import random_uuid |
| 49 | from vllm.utils.async_utils import make_async |
| 50 | from vllm.v1.engine.detokenizer import uses_fast_detokenizer |
| 51 | |
| 52 | logger = init_logger(__name__) |
| 53 | |
| 54 | |
| 55 | class OnlineDerenderer: |
| 56 | def __init__( |
| 57 | self, |
| 58 | model_config: ModelConfig, |
| 59 | renderer: BaseRenderer, |
| 60 | *, |
| 61 | request_logger: RequestLogger | None, |
| 62 | chat_template: str | None, |
| 63 | chat_template_content_format: ChatTemplateContentFormatOption, |
| 64 | trust_request_chat_template: bool = False, |
| 65 | enable_auto_tools: bool = False, |
| 66 | exclude_tools_when_tool_choice_none: bool = False, |
| 67 | tool_parser: str | None = None, |
| 68 | reasoning_parser: str | None = None, |
| 69 | tool_strict_level: str = "auto", |
| 70 | default_chat_template_kwargs: dict[str, Any] | None = None, |
| 71 | log_error_stack: bool = False, |
| 72 | ) -> None: |
| 73 | self.model_config = model_config |
| 74 | self.renderer = renderer |
| 75 | self.request_logger = request_logger |
| 76 | |
| 77 | self.enable_auto_tools = enable_auto_tools |
| 78 | self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none |
| 79 | self.use_harmony = model_config.hf_config.model_type == "gpt_oss" |
| 80 | self.parser: type[Parser] | None = ParserManager.get_parser( |
| 81 | tool_parser_name=tool_parser, |
| 82 | reasoning_parser_name=reasoning_parser, |
| 83 | enable_auto_tools=enable_auto_tools, |
| 84 | tool_strict_level=tool_strict_level, |
| 85 | model_name=model_config.model, |
| 86 | is_harmony=self.use_harmony, |
| 87 | tokenizer=renderer.tokenizer, |
| 88 | ) |
| 89 | |
| 90 | self.chat_template = chat_template |
| 91 | self.chat_template_content_format: ChatTemplateContentFormatOption = ( |
| 92 | chat_template_content_format |
| 93 | ) |
| 94 | self.default_chat_template_kwargs: dict[str, Any] = ( |
| 95 | default_chat_template_kwargs or {} |
| 96 | ) |
| 97 | self.trust_request_chat_template = trust_request_chat_template |
| 98 | |
| 99 | self.log_error_stack = log_error_stack |
| 100 | self.supports_browsing = False |
| 101 | self.supports_code_interpreter = False |
| 102 | |
| 103 | # Detokenization, logprob resolution and parsing are CPU-bound; |
| 104 | # offload them in one hop to keep the event loop responsive. |
| 105 | self._derender_chat_async = make_async( |
| 106 | self._derender_chat, executor=renderer._executor |
| 107 | ) |
| 108 | self._derender_completion_async = make_async( |
| 109 | self._derender_completion, executor=renderer._executor |
| 110 | ) |
| 111 | self._detokenize_delta_async = make_async( |
| 112 | self._detokenize_delta, executor=renderer._executor |
| 113 | ) |
| 114 | # Replay is O(n) per chunk, so it must not run on the event loop. |
| 115 | self._derender_chat_stream_parsed_async = make_async( |
| 116 | self._derender_chat_stream_parsed, executor=renderer._executor |
| 117 | ) |
| 118 | |
| 119 | async def derender_chat( |
| 120 | self, |
| 121 | generate_response: GenerateResponse, |
| 122 | chat_request: ChatCompletionRequest | None = None, |
| 123 | prompt_token_ids: list[int] | None = None, |
| 124 | ) -> list[ChatCompletionResponseChoice]: |
| 125 | return await self._derender_chat_async( |
| 126 | generate_response, chat_request, prompt_token_ids |
| 127 | ) |
| 128 | |
| 129 | def _derender_chat( |
| 130 | self, |
| 131 | generate_response: GenerateResponse, |
| 132 | chat_request: ChatCompletionRequest | None = None, |
| 133 | prompt_token_ids: list[int] | None = None, |
| 134 | ) -> list[ChatCompletionResponseChoice]: |
| 135 | tokenizer = self.renderer.get_tokenizer() |
| 136 | choices: list[ChatCompletionResponseChoice] = [] |
| 137 | |
| 138 | has_parser = self.parser is not None and chat_request is not None |
| 139 | skip_special, spaces_between = _decode_params( |
| 140 | tokenizer, chat_request, preserve_special=has_parser |
| 141 | ) |
| 142 | seed_ids = ( |
| 143 | prompt_token_ids |
| 144 | if prompt_token_ids is not None |
| 145 | else generate_response.prompt_token_ids |
| 146 | ) |
| 147 | # Choices continue the same prompt, so they share one seeded window. |
| 148 | seed_state = _seed_stream_state( |
| 149 | tokenizer, |
| 150 | seed_ids, |
| 151 | skip_special_tokens=skip_special, |
| 152 | ) |
| 153 | |
| 154 | for choice in generate_response.choices: |
| 155 | if not choice.token_ids: |
| 156 | raise ValueError(f"choice {choice.index} has empty or null token_ids") |
| 157 | |
| 158 | resolved_logprobs = ( |
| 159 | _resolve_logprobs(choice.logprobs, tokenizer) |
| 160 | if choice.logprobs is not None |
| 161 | else None |
| 162 | ) |
| 163 | |
| 164 | if has_parser: |
| 165 | assert self.parser is not None and chat_request is not None |
| 166 | # Parser path: decode with special tokens preserved |
| 167 | # so the parser can see markers like </think>, |
| 168 | # <tool_call>, or Harmony channel tokens. |
| 169 | decoded_text, _ = self._detokenize_delta( |
| 170 | tokenizer, |
| 171 | choice.token_ids, |
| 172 | seed_state, |
| 173 | skip_special_tokens=skip_special, |
| 174 | spaces_between_special_tokens=spaces_between, |
| 175 | ) |
| 176 | |
| 177 | chat_template_kwargs: dict[str, Any] = {} |
| 178 | if not self.use_harmony: |
| 179 | chat_template_kwargs = ( |
| 180 | chat_request.build_chat_params( |
| 181 | self.chat_template, |
| 182 | self.chat_template_content_format, |
| 183 | ) |
| 184 | .with_defaults(self.default_chat_template_kwargs) |
| 185 | .chat_template_kwargs |
| 186 | ) |
| 187 | |
| 188 | parser = self.parser( |
| 189 | tokenizer, |
| 190 | chat_request.tools, |
| 191 | chat_template_kwargs=chat_template_kwargs, |
| 192 | model_config=self.model_config, |
| 193 | ) |
| 194 | if generate_response.prompt_token_ids is not None: |
| 195 | parser.set_prompt_token_ids(generate_response.prompt_token_ids) |
| 196 | reasoning, content, tool_calls = parser.parse( |
| 197 | decoded_text, |
| 198 | chat_request, |
| 199 | enable_auto_tools=self.enable_auto_tools, |
| 200 | model_output_token_ids=choice.token_ids, |
| 201 | ) |
| 202 | |
| 203 | if not getattr(chat_request, "include_reasoning", True): |
| 204 | reasoning = None |
| 205 | |
| 206 | tc_items = ( |
| 207 | [ |
| 208 | ToolCall( |
| 209 | id=random_uuid(), |
| 210 | function=tc, |
| 211 | ) |
| 212 | for tc in tool_calls |
| 213 | ] |
| 214 | if tool_calls |
| 215 | else [] |
| 216 | ) |
| 217 | |
| 218 | is_named_tool_choice = ( |
| 219 | type(chat_request.tool_choice) is ChatCompletionNamedToolChoiceParam |
| 220 | ) |
| 221 | is_required_tool_choice = chat_request.tool_choice == "required" |
| 222 | if is_named_tool_choice or is_required_tool_choice: |
| 223 | content = content or "" |
| 224 | |
| 225 | message = ChatMessage( |
| 226 | role="assistant", |
| 227 | reasoning=reasoning, |
| 228 | content=content, |
| 229 | tool_calls=tc_items, |
| 230 | ) |
| 231 | else: |
| 232 | # No parser: plain detokenization honouring the request's |
| 233 | # skip_special_tokens (default True when no request was given). |
| 234 | decoded_text, _ = self._detokenize_delta( |
| 235 | tokenizer, |
| 236 | choice.token_ids, |
| 237 | seed_state, |
| 238 | skip_special_tokens=skip_special, |
| 239 | spaces_between_special_tokens=spaces_between, |
| 240 | ) |
| 241 | message = ChatMessage(role="assistant", content=decoded_text) |
| 242 | |
| 243 | choices.append( |
| 244 | ChatCompletionResponseChoice( |
| 245 | index=choice.index, |
| 246 | message=message, |
| 247 | logprobs=resolved_logprobs, |
| 248 | finish_reason=choice.finish_reason, |
| 249 | ) |
| 250 | ) |
| 251 | |
| 252 | return choices |
| 253 | |
| 254 | def _detokenize_delta( |
| 255 | self, |
| 256 | tokenizer: TokenizerLike, |
| 257 | delta_token_ids: list[int], |
| 258 | state: DerenderStreamState, |
| 259 | skip_special_tokens: bool = True, |
| 260 | spaces_between_special_tokens: bool = True, |
| 261 | ) -> tuple[str, DerenderStreamState]: |
| 262 | """Incrementally detokenize ``delta_token_ids`` from prior stream state. |
| 263 | |
| 264 | Resumes decoding from the offsets carried in ``state`` rather than |
| 265 | replaying token history. ``state.prev_tokens`` holds the trailing decode |
| 266 | window (from ``prefix_offset`` onward) that ``detokenize_incrementally`` |
| 267 | still needs to reproduce any partially read multi-byte character |
| 268 | (tracked by ``read_offset``). The delta tokens are fed straight onto it. |
| 269 | |
| 270 | The window is bounded. ``detokenize_incrementally`` never reads before |
| 271 | ``prefix_offset``, so after each token we trim ``prev_tokens`` to that |
| 272 | tail and rebase the offsets to it. State transport therefore stays |
| 273 | O(window) per chunk instead of re-sending the full token history. |
| 274 | |
| 275 | Args: |
| 276 | tokenizer: The tokenizer to decode with. |
| 277 | delta_token_ids: New token IDs from this generate chunk. |
| 278 | state: Client carried detok state from the previous call. |
| 279 | skip_special_tokens: Passed through to the tokenizer. |
| 280 | spaces_between_special_tokens: Passed through to the tokenizer. |
| 281 | |
| 282 | Returns: |
| 283 | (new_text, updated_state) — the delta text for this chunk and the |
| 284 | state to pass to the next call. |
| 285 | |
| 286 | """ |
| 287 | prev_tokens = list(state.prev_tokens) |
| 288 | prefix_offset = state.prefix_offset |
| 289 | read_offset = state.read_offset |
| 290 | |
| 291 | text_parts: list[str] = [] |
| 292 | for tok_id in delta_token_ids: |
| 293 | # prev_tokens is a (possibly empty) list, never None, so this |
| 294 | # always takes the non first iter path and only consumes |
| 295 | # all_input_ids[-1]. |
| 296 | new_toks, text, prefix_offset, read_offset = detokenize_incrementally( |
| 297 | tokenizer=tokenizer, |
| 298 | all_input_ids=[tok_id], |
| 299 | prev_tokens=prev_tokens, |
| 300 | prefix_offset=prefix_offset, |
| 301 | read_offset=read_offset, |
| 302 | skip_special_tokens=skip_special_tokens, |
| 303 | spaces_between_special_tokens=spaces_between_special_tokens, |
| 304 | ) |
| 305 | # Trim to the tail still readable by detokenize_incrementally |
| 306 | # (everything before prefix_offset is dead) and rebase the |
| 307 | # offsets, so the window stays bounded within a long batch |
| 308 | # decode as well as across chunks. |
| 309 | prev_tokens = (prev_tokens + new_toks)[prefix_offset:] |
| 310 | read_offset -= prefix_offset |
| 311 | prefix_offset = 0 |
| 312 | text_parts.append(text) |
| 313 | |
| 314 | updated_state = state.model_copy( |
| 315 | update={ |
| 316 | "prev_tokens": prev_tokens, |
| 317 | "prefix_offset": prefix_offset, |
| 318 | "read_offset": read_offset, |
| 319 | } |
| 320 | ) |
| 321 | return "".join(text_parts), updated_state |
| 322 | |
| 323 | async def derender_chat_stream( |
| 324 | self, |
| 325 | model: str, |
| 326 | generate_chunk: GenerateStreamResponse, |
| 327 | state: DerenderStreamState | None = None, |
| 328 | chat_request: ChatCompletionRequest | None = None, |
| 329 | prompt_tokens: int | None = None, |
| 330 | prompt_token_ids: list[int] | None = None, |
| 331 | ) -> tuple[ChatCompletionStreamResponse, DerenderStreamState]: |
| 332 | """Process one GenerateStreamResponse chunk for streaming chat derender. |
| 333 | |
| 334 | Unlike OpenAI's API, which always emits `role: "assistant"` on the |
| 335 | very first chunk, this emits it on the first chunk with a non empty |
| 336 | `choices` list. A leading usage only chunk therefore defers the |
| 337 | role to the following content chunk instead of sending an empty |
| 338 | role only delta. |
| 339 | |
| 340 | Args: |
| 341 | model: Model name for the response object. |
| 342 | generate_chunk: One SSE chunk from `/inference/v1/generate`. |
| 343 | state: Client carried detok state (`None` for first call). |
| 344 | chat_request: Original ChatCompletionRequest from `/render`. |
| 345 | Required when a reasoning or tool parser is configured |
| 346 | (validated by the caller — see `ServingDerender`) because |
| 347 | plain detokenization would leak raw parser markup into `content`. |
| 348 | prompt_tokens: Prompt token count for the usage chunk. |
| 349 | prompt_token_ids: Prompt token IDs. Required on the parser path |
| 350 | (validated by the caller) and optional otherwise, falling |
| 351 | back to ``generate_chunk.prompt_token_ids``. See |
| 352 | `DerenderChatStreamRequest.prompt_token_ids`. |
| 353 | |
| 354 | Returns: |
| 355 | (chunk, updated_state) — the derendered SSE chunk and the state |
| 356 | the client must pass to the next call. |
| 357 | |
| 358 | """ |
| 359 | # A single DerenderStreamState is threaded through every choice in |
| 360 | # this chunk. Correct only when there is at most one choice per SSE |
| 361 | # event (n=1, one call per index), as the streaming derender |
| 362 | # protocol assumes. Multiple choices sharing one chunk would corrupt |
| 363 | # each other's detok/parser state. |
| 364 | if len(generate_chunk.choices) > 1: |
| 365 | raise ValueError( |
| 366 | "derender_chat_stream expects at most one choice per chunk" |
| 367 | ) |
| 368 | |
| 369 | parser_cls = self.parser |
| 370 | if parser_cls is not None: |
| 371 | # Fail (mirrors ServingDerender's pre-check) because a parser |
| 372 | # configured model must never fall through to plain detok or |
| 373 | # reasoning/tool markup would leak into `delta.content`. |
| 374 | if chat_request is None: |
| 375 | raise ValueError( |
| 376 | "chat_request is required for streaming chat derender " |
| 377 | "when a tool or reasoning parser is configured" |
| 378 | ) |
| 379 | return await self._derender_chat_stream_parsed_async( |
| 380 | parser_cls, |
| 381 | model, |
| 382 | generate_chunk, |
| 383 | state if state is not None else DerenderStreamState(), |
| 384 | chat_request, |
| 385 | prompt_tokens, |
| 386 | prompt_token_ids, |
| 387 | ) |
| 388 | |
| 389 | tokenizer = self.renderer.get_tokenizer() |
| 390 | skip_special, spaces_between = _decode_params(tokenizer, chat_request) |
| 391 | # Seed on the first chunk only. A carried state already has the prompt. |
| 392 | if state is None: |
| 393 | state = _seed_stream_state( |
| 394 | tokenizer, |
| 395 | prompt_token_ids |
| 396 | if prompt_token_ids is not None |
| 397 | else generate_chunk.prompt_token_ids, |
| 398 | skip_special_tokens=skip_special, |
| 399 | ) |
| 400 | stream_choices: list[ChatCompletionResponseStreamChoice] = [] |
| 401 | updated_state = state |
| 402 | |
| 403 | for choice in generate_chunk.choices: |
| 404 | delta_tids = choice.token_ids or [] |
| 405 | new_text, updated_state = await self._detokenize_delta_async( |
| 406 | tokenizer, |
| 407 | delta_tids, |
| 408 | updated_state, |
| 409 | skip_special_tokens=skip_special, |
| 410 | spaces_between_special_tokens=spaces_between, |
| 411 | ) |
| 412 | |
| 413 | # NOTE: parser-configured servers dispatch to |
| 414 | # _derender_chat_stream_parsed above and never reach this plain |
| 415 | # path. That parsed path does not resolve logprobs yet; when it |
| 416 | # does, mirror the generate chat streaming path, which suppresses |
| 417 | # logprobs entirely when a parser is configured and reasoning is |
| 418 | # hidden, because decoded logprob token text would leak hidden |
| 419 | # reasoning. |
| 420 | resolved_logprobs = None |
| 421 | if choice.logprobs is not None: |
| 422 | resolved_logprobs = _resolve_logprobs( |
| 423 | choice.logprobs, |
| 424 | tokenizer, |
| 425 | initial_context_token_ids=state.logprob_context_token_ids, |
| 426 | ) |
| 427 | |
| 428 | include_role = not updated_state.role_sent |
| 429 | updated_state = updated_state.model_copy( |
| 430 | update={ |
| 431 | "role_sent": True, |
| 432 | "logprob_context_token_ids": _logprob_context_tail( |
| 433 | state.logprob_context_token_ids, delta_tids |
| 434 | ), |
| 435 | } |
| 436 | ) |
| 437 | |
| 438 | delta = DeltaMessage( |
| 439 | role="assistant" if include_role else None, |
| 440 | content=new_text if new_text else None, |
| 441 | ) |
| 442 | stream_choices.append( |
| 443 | ChatCompletionResponseStreamChoice( |
| 444 | index=choice.index, |
| 445 | delta=delta, |
| 446 | logprobs=resolved_logprobs, |
| 447 | finish_reason=choice.finish_reason, |
| 448 | ) |
| 449 | ) |
| 450 | |
| 451 | usage: UsageInfo | None = None |
| 452 | if generate_chunk.usage is not None: |
| 453 | u = generate_chunk.usage |
| 454 | pt = prompt_tokens if prompt_tokens is not None else (u.prompt_tokens or 0) |
| 455 | ct = u.completion_tokens or 0 |
| 456 | usage = UsageInfo( |
| 457 | prompt_tokens=pt, |
| 458 | completion_tokens=ct, |
| 459 | total_tokens=pt + ct, |
| 460 | ) |
| 461 | |
| 462 | chunk = ChatCompletionStreamResponse( |
| 463 | id=generate_chunk.request_id, |
| 464 | model=model, |
| 465 | choices=stream_choices, |
| 466 | usage=usage, |
| 467 | metrics=generate_chunk.metrics, |
| 468 | ) |
| 469 | return chunk, updated_state |
| 470 | |
| 471 | def _derender_chat_stream_parsed( |
| 472 | self, |
| 473 | parser_cls: type[Parser], |
| 474 | model: str, |
| 475 | generate_chunk: GenerateStreamResponse, |
| 476 | state: DerenderStreamState, |
| 477 | chat_request: ChatCompletionRequest, |
| 478 | prompt_tokens: int | None, |
| 479 | prompt_token_ids: list[int] | None, |
| 480 | ) -> tuple[ChatCompletionStreamResponse, DerenderStreamState]: |
| 481 | """Parser path for streaming chat derender: replay + `parse_delta`. |
| 482 | |
| 483 | Parser internal state (buffered markup, reasoning/tool phase, etc.) |
| 484 | cannot be serialized into `DerenderStreamState`, so each call |
| 485 | builds a fresh parser and replays every prior output token through |
| 486 | `parse_delta` (discarding the result) before processing this |
| 487 | chunk's tokens for real. |
| 488 | |
| 489 | Every chunk, replayed or live, goes through one `parse_delta` call |
| 490 | with the tokens it arrived with (more than one under e.g. |
| 491 | speculative decoding). Replay recovers those boundaries from |
| 492 | `state.output_chunk_lens`, so the rebuilt parser state matches |
| 493 | standard serving, which calls `parse_delta` once per engine step. |
| 494 | Text comes from a fresh incremental detokenizer with special tokens |
| 495 | preserved (``skip_special_tokens=False``), seeded from the prompt |
| 496 | tail on every call since replay starts from scratch. |
| 497 | """ |
| 498 | tokenizer = self.renderer.get_tokenizer() |
| 499 | |
| 500 | chat_template_kwargs: dict[str, Any] = {} |
| 501 | if not self.use_harmony: |
| 502 | chat_template_kwargs = ( |
| 503 | chat_request.build_chat_params( |
| 504 | self.chat_template, |
| 505 | self.chat_template_content_format, |
| 506 | ) |
| 507 | .with_defaults(self.default_chat_template_kwargs) |
| 508 | .chat_template_kwargs |
| 509 | ) |
| 510 | |
| 511 | parser = parser_cls( |
| 512 | tokenizer, |
| 513 | chat_request.tools, |
| 514 | chat_template_kwargs=chat_template_kwargs, |
| 515 | model_config=self.model_config, |
| 516 | ) |
| 517 | |
| 518 | # Ephemeral incremental detok window, local to this call. Threaded |
| 519 | # across both the replay and current chunk phases (via `nonlocal`) |
| 520 | # so multi-byte characters split across that boundary still decode |
| 521 | # correctly. Discarded once the call returns. |
| 522 | skip_special, spaces_between = _decode_params( |
| 523 | tokenizer, chat_request, preserve_special=True |
| 524 | ) |
| 525 | detok_state = _seed_stream_state( |
| 526 | tokenizer, prompt_token_ids, skip_special_tokens=skip_special |
| 527 | ) |
| 528 | |
| 529 | def _replay(token_ids: list[int], chunk_lens: list[int]) -> None: |
| 530 | """Replay prior chunks through `parse_delta` to rebuild parser |
| 531 | state, discarding the result. Never `finished` since that only |
| 532 | applies to the current chunk. |
| 533 | """ |
| 534 | nonlocal detok_state |
| 535 | start = 0 |
| 536 | for chunk_len in chunk_lens: |
| 537 | chunk = token_ids[start : start + chunk_len] |
| 538 | start += chunk_len |
| 539 | text, detok_state = self._detokenize_delta( |
| 540 | tokenizer, |
| 541 | chunk, |
| 542 | detok_state, |
| 543 | skip_special_tokens=skip_special, |
| 544 | spaces_between_special_tokens=spaces_between, |
| 545 | ) |
| 546 | parser.parse_delta( |
| 547 | text, |
| 548 | chunk, |
| 549 | chat_request, |
| 550 | prompt_token_ids=prompt_token_ids, |
| 551 | finished=False, |
| 552 | ) |
| 553 | |
| 554 | # Replay history to reconstruct parser state. The result is thrown |
| 555 | # away and only the current chunk's emission goes to the client. |
| 556 | _replay(state.output_token_ids, state.output_chunk_lens) |
| 557 | |
| 558 | stream_choices: list[ChatCompletionResponseStreamChoice] = [] |
| 559 | output_token_ids = list(state.output_token_ids) |
| 560 | output_chunk_lens = list(state.output_chunk_lens) |
| 561 | role_sent = state.role_sent |
| 562 | tools_streamed = state.tools_streamed |
| 563 | last_tool_call_ids = list(state.last_tool_call_ids) |
| 564 | |
| 565 | # At most one choice: the caller (derender_chat_stream) already |
| 566 | # rejects >1 before dispatching here. role_sent/tools_streamed/ |
| 567 | # output_token_ids below are updated for that single choice, not |
| 568 | # accumulated across choices. Looping over generate_chunk.choices |
| 569 | # could silently corrupt output if chunks were ever allowed to |
| 570 | # contain multiple choices. |
| 571 | if generate_chunk.choices: |
| 572 | choice = generate_chunk.choices[0] |
| 573 | delta_tids = choice.token_ids or [] |
| 574 | is_finished = choice.finish_reason is not None |
| 575 | |
| 576 | if delta_tids: |
| 577 | # One parse_delta call for the whole chunk (producer |
| 578 | # granularity), not one per token. See the granularity note |
| 579 | # in this method's docstring. |
| 580 | text, detok_state = self._detokenize_delta( |
| 581 | tokenizer, |
| 582 | delta_tids, |
| 583 | detok_state, |
| 584 | skip_special_tokens=skip_special, |
| 585 | spaces_between_special_tokens=spaces_between, |
| 586 | ) |
| 587 | delta_message = parser.parse_delta( |
| 588 | text, |
| 589 | delta_tids, |
| 590 | chat_request, |
| 591 | prompt_token_ids=prompt_token_ids, |
| 592 | finished=is_finished, |
| 593 | ) |
| 594 | elif is_finished: |
| 595 | # Finish only chunk (no new tokens). Still flush any |
| 596 | # buffered tool call arguments. |
| 597 | delta_message = parser.parse_delta( |
| 598 | "", |
| 599 | [], |
| 600 | chat_request, |
| 601 | prompt_token_ids=prompt_token_ids, |
| 602 | finished=True, |
| 603 | ) |
| 604 | else: |
| 605 | delta_message = None |
| 606 | |
| 607 | if delta_tids: |
| 608 | output_token_ids.extend(delta_tids) |
| 609 | output_chunk_lens.append(len(delta_tids)) |
| 610 | |
| 611 | if delta_message is None: |
| 612 | delta_message = DeltaMessage() |
| 613 | |
| 614 | if delta_message.tool_calls: |
| 615 | tools_streamed = True |
| 616 | for tc in delta_message.tool_calls: |
| 617 | if tc.id is None: |
| 618 | continue |
| 619 | if tc.index < len(last_tool_call_ids): |
| 620 | # Pin: reuse the ID already recorded for this index |
| 621 | # rather than one a from scratch replay regenerated. |
| 622 | # Real trigger not just defensive with |
| 623 | # tool_choice="required", |
| 624 | # extract_required_tool_call_streaming resets |
| 625 | # function_name_returned to False whenever the |
| 626 | # partial JSON transiently fails to parse which |
| 627 | # re-emits id+name for the same index on replay. |
| 628 | tc.id = last_tool_call_ids[tc.index] |
| 629 | else: |
| 630 | last_tool_call_ids.append(tc.id) |
| 631 | |
| 632 | if not role_sent: |
| 633 | delta_message.role = "assistant" |
| 634 | role_sent = True |
| 635 | |
| 636 | finish_reason = choice.finish_reason |
| 637 | if finish_reason is not None: |
| 638 | is_named_tool_choice = ( |
| 639 | type(chat_request.tool_choice) is ChatCompletionNamedToolChoiceParam |
| 640 | ) |
| 641 | if tools_streamed and not is_named_tool_choice: |
| 642 | finish_reason = "tool_calls" |
| 643 | |
| 644 | stream_choice = ChatCompletionResponseStreamChoice( |
| 645 | index=choice.index, |
| 646 | delta=delta_message, |
| 647 | finish_reason=finish_reason, |
| 648 | ) |
| 649 | stream_choices.append( |
| 650 | maybe_filter_parallel_tool_calls(stream_choice, chat_request) |
| 651 | ) |
| 652 | |
| 653 | updated_state = state.model_copy( |
| 654 | update={ |
| 655 | "output_token_ids": output_token_ids, |
| 656 | "output_chunk_lens": output_chunk_lens, |
| 657 | "role_sent": role_sent, |
| 658 | "tools_streamed": tools_streamed, |
| 659 | "last_tool_call_ids": last_tool_call_ids, |
| 660 | } |
| 661 | ) |
| 662 | |
| 663 | usage: UsageInfo | None = None |
| 664 | if generate_chunk.usage is not None: |
| 665 | u = generate_chunk.usage |
| 666 | pt = prompt_tokens if prompt_tokens is not None else (u.prompt_tokens or 0) |
| 667 | ct = u.completion_tokens or 0 |
| 668 | usage = UsageInfo( |
| 669 | prompt_tokens=pt, |
| 670 | completion_tokens=ct, |
| 671 | total_tokens=pt + ct, |
| 672 | ) |
| 673 | |
| 674 | chunk = ChatCompletionStreamResponse( |
| 675 | id=generate_chunk.request_id, |
| 676 | model=model, |
| 677 | choices=stream_choices, |
| 678 | usage=usage, |
| 679 | metrics=generate_chunk.metrics, |
| 680 | ) |
| 681 | return chunk, updated_state |
| 682 | |
| 683 | async def derender_completion( |
| 684 | self, |
| 685 | generate_responses: list[GenerateResponse], |
| 686 | prompt_tokens: list[int] | None = None, |
| 687 | completion_request: CompletionRequest | None = None, |
| 688 | prompt_token_ids: list[list[int] | None] | None = None, |
| 689 | ) -> tuple[list[CompletionResponseChoice], int, int]: |
| 690 | return await self._derender_completion_async( |
| 691 | generate_responses, prompt_tokens, completion_request, prompt_token_ids |
| 692 | ) |
| 693 | |
| 694 | def _derender_completion( |
| 695 | self, |
| 696 | generate_responses: list[GenerateResponse], |
| 697 | prompt_tokens: list[int] | None = None, |
| 698 | completion_request: CompletionRequest | None = None, |
| 699 | prompt_token_ids: list[list[int] | None] | None = None, |
| 700 | ) -> tuple[list[CompletionResponseChoice], int, int]: |
| 701 | n = len(generate_responses) |
| 702 | prompt_tokens_list: list[int] = ( |
| 703 | prompt_tokens if prompt_tokens is not None else [0] * n |
| 704 | ) |
| 705 | prompt_token_ids_list: list[list[int] | None] = ( |
| 706 | prompt_token_ids if prompt_token_ids is not None else [None] * n |
| 707 | ) |
| 708 | |
| 709 | tokenizer = self.renderer.get_tokenizer() |
| 710 | skip_special, spaces_between = _decode_params(tokenizer, completion_request) |
| 711 | choices: list[CompletionResponseChoice] = [] |
| 712 | total_prompt_tokens = 0 |
| 713 | total_completion_tokens = 0 |
| 714 | index = 0 |
| 715 | |
| 716 | for gen, pt, seed_ids_override in zip( |
| 717 | generate_responses, prompt_tokens_list, prompt_token_ids_list |
| 718 | ): |
| 719 | seed_ids = ( |
| 720 | seed_ids_override |
| 721 | if seed_ids_override is not None |
| 722 | else gen.prompt_token_ids |
| 723 | ) |
| 724 | seed_state = _seed_stream_state( |
| 725 | tokenizer, seed_ids, skip_special_tokens=skip_special |
| 726 | ) |
| 727 | for choice in gen.choices: |
| 728 | if not choice.token_ids: |
| 729 | raise ValueError( |
| 730 | f"choice {choice.index} in response {gen.request_id} " |
| 731 | "has empty or null token_ids" |
| 732 | ) |
| 733 | |
| 734 | decoded_text, _ = self._detokenize_delta( |
| 735 | tokenizer, |
| 736 | choice.token_ids, |
| 737 | seed_state, |
| 738 | skip_special_tokens=skip_special, |
| 739 | spaces_between_special_tokens=spaces_between, |
| 740 | ) |
| 741 | completion_logprobs = None |
| 742 | if choice.logprobs is not None: |
| 743 | resolved = _resolve_logprobs(choice.logprobs, tokenizer) |
| 744 | completion_logprobs = _convert_chat_logprobs_to_completion_logprobs( |
| 745 | resolved |
| 746 | ) |
| 747 | choices.append( |
| 748 | CompletionResponseChoice( |
| 749 | index=index, |
| 750 | text=decoded_text, |
| 751 | finish_reason=choice.finish_reason, |
| 752 | logprobs=completion_logprobs, |
| 753 | ) |
| 754 | ) |
| 755 | total_completion_tokens += len(choice.token_ids) |
| 756 | index += 1 |
| 757 | total_prompt_tokens += pt |
| 758 | |
| 759 | return choices, total_prompt_tokens, total_completion_tokens |
| 760 | |
| 761 | async def derender_completion_stream( |
| 762 | self, |
| 763 | model: str, |
| 764 | generate_chunk: GenerateStreamResponse, |
| 765 | state: DerenderStreamState | None = None, |
| 766 | prompt_tokens: int | None = None, |
| 767 | completion_request: CompletionRequest | None = None, |
| 768 | prompt_token_ids: list[int] | None = None, |
| 769 | ) -> tuple[CompletionStreamResponse, DerenderStreamState]: |
| 770 | """Process one GenerateStreamResponse chunk for streaming completions. |
| 771 | |
| 772 | Each call takes one SSE chunk from ``/inference/v1/generate`` plus the |
| 773 | client carried ``stream_state`` and returns a ``CompletionStreamResponse`` |
| 774 | chunk and the updated state. |
| 775 | |
| 776 | The generate stream emits one choice per SSE event, so this method |
| 777 | processes one output sequence at a time. For ``n > 1`` the client |
| 778 | maintains one ``DerenderStreamState`` per ``choice.index``. |
| 779 | |
| 780 | Args: |
| 781 | model: Model name for the response object. |
| 782 | generate_chunk: One SSE chunk from ``/inference/v1/generate``. |
| 783 | state: Client carried detok state (``None`` → first call). |
| 784 | prompt_tokens: Prompt token count for usage (from the render step). |
| 785 | completion_request: Original CompletionRequest from ``/render``; |
| 786 | supplies ``skip_special_tokens``. |
| 787 | prompt_token_ids: Seeds the first chunk's decode. Falls back to |
| 788 | ``generate_chunk.prompt_token_ids``. |
| 789 | |
| 790 | Returns: |
| 791 | (chunk, updated_state) — the derendered chunk and updated state. |
| 792 | |
| 793 | """ |
| 794 | # See the equivalent check in derender_chat_stream: a single |
| 795 | # DerenderStreamState is threaded through every choice in this |
| 796 | # chunk, so more than one choice per chunk would corrupt the |
| 797 | # detok window across choices. |
| 798 | if len(generate_chunk.choices) > 1: |
| 799 | raise ValueError( |
| 800 | "derender_completion_stream expects at most one choice per chunk" |
| 801 | ) |
| 802 | |
| 803 | tokenizer = self.renderer.get_tokenizer() |
| 804 | skip_special, spaces_between = _decode_params(tokenizer, completion_request) |
| 805 | # Seed on the first chunk only. A carried state already has the prompt. |
| 806 | if state is None: |
| 807 | state = _seed_stream_state( |
| 808 | tokenizer, |
| 809 | prompt_token_ids |
| 810 | if prompt_token_ids is not None |
| 811 | else generate_chunk.prompt_token_ids, |
| 812 | skip_special_tokens=skip_special, |
| 813 | ) |
| 814 | stream_choices: list[CompletionResponseStreamChoice] = [] |
| 815 | updated_state = state |
| 816 | |
| 817 | for choice in generate_chunk.choices: |
| 818 | delta_tids = choice.token_ids or [] |
| 819 | new_text, updated_state = await self._detokenize_delta_async( |
| 820 | tokenizer, |
| 821 | delta_tids, |
| 822 | updated_state, |
| 823 | skip_special_tokens=skip_special, |
| 824 | spaces_between_special_tokens=spaces_between, |
| 825 | ) |
| 826 | |
| 827 | completion_logprobs = None |
| 828 | if choice.logprobs is not None: |
| 829 | resolved = _resolve_logprobs( |
| 830 | choice.logprobs, |
| 831 | tokenizer, |
| 832 | initial_context_token_ids=state.logprob_context_token_ids, |
| 833 | ) |
| 834 | completion_logprobs = _convert_chat_logprobs_to_completion_logprobs( |
| 835 | resolved, initial_text_offset=state.logprob_text_offset |
| 836 | ) |
| 837 | |
| 838 | updated_state = updated_state.model_copy( |
| 839 | update={ |
| 840 | "logprob_context_token_ids": _logprob_context_tail( |
| 841 | state.logprob_context_token_ids, delta_tids |
| 842 | ), |
| 843 | "logprob_text_offset": state.logprob_text_offset + len(new_text), |
| 844 | } |
| 845 | ) |
| 846 | |
| 847 | stream_choices.append( |
| 848 | CompletionResponseStreamChoice( |
| 849 | index=choice.index, |
| 850 | text=new_text, |
| 851 | logprobs=completion_logprobs, |
| 852 | finish_reason=choice.finish_reason, |
| 853 | ) |
| 854 | ) |
| 855 | |
| 856 | usage: UsageInfo | None = None |
| 857 | if generate_chunk.usage is not None: |
| 858 | u = generate_chunk.usage |
| 859 | pt = prompt_tokens if prompt_tokens is not None else (u.prompt_tokens or 0) |
| 860 | ct = u.completion_tokens or 0 |
| 861 | usage = UsageInfo( |
| 862 | prompt_tokens=pt, |
| 863 | completion_tokens=ct, |
| 864 | total_tokens=pt + ct, |
| 865 | ) |
| 866 | |
| 867 | chunk = CompletionStreamResponse( |
| 868 | id=generate_chunk.request_id, |
| 869 | model=model, |
| 870 | choices=stream_choices, |
| 871 | usage=usage, |
| 872 | metrics=generate_chunk.metrics, |
| 873 | ) |
| 874 | return chunk, updated_state |
| 875 | |
| 876 | |
| 877 | # Number of preceding sampled token IDs `_correct_decoded_token` reads to |
| 878 | # repair U+FFFD from byte-fallback tokenization. |
| 879 | _LOGPROB_CONTEXT_WINDOW = 4 |
| 880 | |
| 881 | |
| 882 | def _logprob_context_tail( |
| 883 | context_token_ids: list[int], delta_token_ids: list[int] |
| 884 | ) -> list[int]: |
| 885 | """Advance the carried logprob context by this chunk's sampled tokens.""" |
| 886 | return (list(context_token_ids) + list(delta_token_ids))[-_LOGPROB_CONTEXT_WINDOW:] |
| 887 | |
| 888 | |
| 889 | def _decode_params( |
| 890 | tokenizer: TokenizerLike, |
| 891 | request: ChatCompletionRequest | CompletionRequest | None, |
| 892 | preserve_special: bool = False, |
| 893 | ) -> tuple[bool, bool]: |
| 894 | """Derive `(skip_special_tokens, spaces_between_special_tokens)` the way |
| 895 | the engine does (`IncrementalDetokenizer`). |
| 896 | |
| 897 | Args: |
| 898 | tokenizer: Picks the engine's fast or slow detokenizer rules. |
| 899 | request: The original request, if the caller supplied one. |
| 900 | preserve_special: Keep special tokens so a parser can see markers. |
| 901 | The serving side does this via `adjust_request`. |
| 902 | |
| 903 | """ |
| 904 | skip_special = ( |
| 905 | False |
| 906 | if preserve_special |
| 907 | else (request.skip_special_tokens if request is not None else True) |
| 908 | ) |
| 909 | spaces_between = ( |
| 910 | request.spaces_between_special_tokens if request is not None else True |
| 911 | ) |
| 912 | # Only the fast detokenizer forces spaces on when skipping special tokens. |
| 913 | if skip_special and uses_fast_detokenizer(tokenizer): |
| 914 | spaces_between = True |
| 915 | return skip_special, spaces_between |
| 916 | |
| 917 | |
| 918 | def _seed_stream_state( |
| 919 | tokenizer: TokenizerLike, |
| 920 | prompt_token_ids: list[int] | None, |
| 921 | skip_special_tokens: bool, |
| 922 | ) -> DerenderStreamState: |
| 923 | """Build the initial decode state from the prompt tail, the same way the |
| 924 | engine primes its incremental detokenizer. Without it, Metaspace |
| 925 | tokenizers drop the first output token's leading space. |
| 926 | |
| 927 | Returns an empty state when `prompt_token_ids` is empty or omitted. |
| 928 | """ |
| 929 | if not prompt_token_ids: |
| 930 | if prompt_token_ids is None and get_leading_space_marker(tokenizer) is not None: |
| 931 | logger.warning_once( |
| 932 | "derender got no prompt_token_ids, so the first output token " |
| 933 | "may lose its leading space compared to the coupled endpoint. " |
| 934 | "Pass prompt_token_ids from the render step to avoid this." |
| 935 | ) |
| 936 | return DerenderStreamState() |
| 937 | |
| 938 | prev_tokens, prefix_offset, read_offset = convert_prompt_ids_to_tokens( |
| 939 | tokenizer, prompt_token_ids, skip_special_tokens=skip_special_tokens |
| 940 | ) |
| 941 | return DerenderStreamState( |
| 942 | prev_tokens=prev_tokens, |
| 943 | prefix_offset=prefix_offset, |
| 944 | read_offset=read_offset, |
| 945 | ) |
| 946 | |
| 947 | |
| 948 | def _parse_token_id_placeholder(token: str) -> int | None: |
| 949 | """Extract token ID from a 'token_id:N' placeholder string.""" |
| 950 | if not token.startswith("token_id:"): |
| 951 | return None |
| 952 | try: |
| 953 | return int(token[len("token_id:") :]) |
| 954 | except ValueError: |
| 955 | return None |
| 956 | |
| 957 | |
| 958 | def _correct_decoded_token( |
| 959 | token_id: int, context_token_ids: list[int], tokenizer: TokenizerLike |
| 960 | ) -> str: |
| 961 | """Use preceding tokens as context to fix U+FFFD from byte-fallback. |
| 962 | |
| 963 | Mirrors LogprobsProcessor._correct_decoded_token in v1/engine/logprobs.py. |
| 964 | """ |
| 965 | max_ctx = min(len(context_token_ids), 4) |
| 966 | |
| 967 | for num_ctx in range(1, max_ctx + 1): |
| 968 | context = context_token_ids[-num_ctx:] |
| 969 | full_decoded = tokenizer.decode(context + [token_id]) |
| 970 | |
| 971 | if full_decoded.endswith("�"): |
| 972 | continue |
| 973 | |
| 974 | clean_end = len(context) |
| 975 | for j in range(len(context) - 1, -1, -1): |
| 976 | if tokenizer.decode([context[j]]).endswith("�"): |
| 977 | clean_end = j |
| 978 | else: |
| 979 | break |
| 980 | |
| 981 | clean_prefix = tokenizer.decode(context[:clean_end]) if clean_end > 0 else "" |
| 982 | |
| 983 | if full_decoded.startswith(clean_prefix): |
| 984 | return full_decoded[len(clean_prefix) :] |
| 985 | |
| 986 | common_len = 0 |
| 987 | for a, b in zip(clean_prefix, full_decoded): |
| 988 | if a != b: |
| 989 | break |
| 990 | common_len += 1 |
| 991 | return full_decoded[common_len:] |
| 992 | |
| 993 | return "" |
| 994 | |
| 995 | |
| 996 | def _resolve_logprobs( |
| 997 | logprobs: ChatCompletionLogProbs, |
| 998 | tokenizer: TokenizerLike, |
| 999 | initial_context_token_ids: Sequence[int] = (), |
| 1000 | ) -> ChatCompletionLogProbs: |
| 1001 | """Resolve token_id:N placeholders in a ChatCompletionLogProbs object. |
| 1002 | |
| 1003 | ``initial_context_token_ids`` seeds the byte-fallback correction context |
| 1004 | with sampled IDs from preceding chunks (streaming), so multi-byte |
| 1005 | characters split across chunk boundaries still resolve. |
| 1006 | """ |
| 1007 | if logprobs.content is None: |
| 1008 | return logprobs |
| 1009 | |
| 1010 | context_token_ids: list[int] = list(initial_context_token_ids) |
| 1011 | resolved_content = [] |
| 1012 | |
| 1013 | for entry in logprobs.content: |
| 1014 | token_str, token_bytes = resolve_token_id_placeholder(entry.token, tokenizer) |
| 1015 | sampled_id = _parse_token_id_placeholder(entry.token) |
| 1016 | |
| 1017 | if token_str.endswith("�") and sampled_id is not None: |
| 1018 | token_str = _correct_decoded_token(sampled_id, context_token_ids, tokenizer) |
| 1019 | token_bytes = list(token_str.encode("utf-8")) |
| 1020 | |
| 1021 | resolved_top = [] |
| 1022 | for top in entry.top_logprobs: |
| 1023 | top_str, top_bytes = resolve_token_id_placeholder(top.token, tokenizer) |
| 1024 | top_id = _parse_token_id_placeholder(top.token) |
| 1025 | if top_str.endswith("�") and top_id is not None: |
| 1026 | top_str = _correct_decoded_token(top_id, context_token_ids, tokenizer) |
| 1027 | top_bytes = list(top_str.encode("utf-8")) |
| 1028 | resolved_top.append( |
| 1029 | top.model_copy(update={"token": top_str, "bytes": top_bytes}) |
| 1030 | ) |
| 1031 | |
| 1032 | resolved_content.append( |
| 1033 | entry.model_copy( |
| 1034 | update={ |
| 1035 | "token": token_str, |
| 1036 | "bytes": token_bytes, |
| 1037 | "top_logprobs": resolved_top, |
| 1038 | } |
| 1039 | ) |
| 1040 | ) |
| 1041 | |
| 1042 | if sampled_id is not None: |
| 1043 | context_token_ids.append(sampled_id) |
| 1044 | |
| 1045 | return ChatCompletionLogProbs(content=resolved_content) |
| 1046 | |
| 1047 | |
| 1048 | def _convert_chat_logprobs_to_completion_logprobs( |
| 1049 | logprobs: ChatCompletionLogProbs, |
| 1050 | initial_text_offset: int = 0, |
| 1051 | ) -> CompletionLogProbs: |
| 1052 | """Convert ChatCompletionLogProbs (per-token objects) to CompletionLogProbs |
| 1053 | (parallel flat lists) as required by the /v1/completions response schema. |
| 1054 | |
| 1055 | ``initial_text_offset`` keeps ``text_offset`` absolute across streaming |
| 1056 | chunks, mirroring the generate streaming path. |
| 1057 | """ |
| 1058 | if logprobs.content is None: |
| 1059 | return CompletionLogProbs() |
| 1060 | |
| 1061 | tokens: list[str] = [] |
| 1062 | token_logprobs: list[float | None] = [] |
| 1063 | top_logprobs_list: list[dict[str, float] | None] = [] |
| 1064 | text_offset: list[int] = [] |
| 1065 | |
| 1066 | offset = initial_text_offset |
| 1067 | for entry in logprobs.content: |
| 1068 | text_offset.append(offset) |
| 1069 | tokens.append(entry.token) |
| 1070 | token_logprobs.append(entry.logprob) |
| 1071 | top_logprobs_list.append( |
| 1072 | {t.token: t.logprob for t in entry.top_logprobs} |
| 1073 | if entry.top_logprobs |
| 1074 | else None |
| 1075 | ) |
| 1076 | offset += len(entry.token) |
| 1077 | |
| 1078 | return CompletionLogProbs( |
| 1079 | text_offset=text_offset, |
| 1080 | token_logprobs=token_logprobs, |
| 1081 | tokens=tokens, |
| 1082 | top_logprobs=top_logprobs_list, |
| 1083 | ) |