| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | from __future__ import annotations |
| 5 | |
| 6 | from dataclasses import dataclass |
| 7 | |
| 8 | import torch |
| 9 | |
| 10 | from vllm.v1.sample.logits_processor import LogitsProcessors |
| 11 | from vllm.v1.sample.thinking_budget_state import ThinkingBudgetStateHolder |
| 12 | |
| 13 | |
| 14 | @dataclass |
| 15 | class SamplingMetadata: |
| 16 | temperature: torch.Tensor | None |
| 17 | all_greedy: bool |
| 18 | all_random: bool |
| 19 | |
| 20 | top_p: torch.Tensor | None |
| 21 | top_k: torch.Tensor | None |
| 22 | |
| 23 | generators: dict[int, torch.Generator] |
| 24 | |
| 25 | # None means no logprobs, 0 means sampled token logprobs only |
| 26 | max_num_logprobs: int | None |
| 27 | |
| 28 | no_penalties: bool |
| 29 | prompt_token_ids: torch.Tensor | None |
| 30 | frequency_penalties: torch.Tensor |
| 31 | presence_penalties: torch.Tensor |
| 32 | repetition_penalties: torch.Tensor |
| 33 | |
| 34 | output_token_ids: list[list[int]] |
| 35 | |
| 36 | # `allowed_token_ids_mask` is a 2D bool tensor of shape (max batch size, |
| 37 | # vocab size). |
| 38 | allowed_token_ids_mask: torch.Tensor | None |
| 39 | |
| 40 | # req_index -> bad_words_token_ids |
| 41 | bad_words_token_ids: dict[int, list[list[int]]] |
| 42 | |
| 43 | # Loaded logits processors |
| 44 | logitsprocs: LogitsProcessors |
| 45 | |
| 46 | # Specific token IDs to compute logprobs for (more efficient than full vocab) |
| 47 | # When set, logprobs are computed only for these token IDs using gather |
| 48 | # req_index -> list of token IDs to get logprobs for |
| 49 | logprob_token_ids: dict[int, list[int]] | None = None |
| 50 | |
| 51 | # Speculative token ids |
| 52 | spec_token_ids: list[list[int]] | None = None |
| 53 | # When non-None, use ``holder.has_tracked_requests()`` to see if this batch applies |
| 54 | # thinking-token-budget logits (holder may exist with an empty tracking set). |
| 55 | thinking_budget_state_holder: ThinkingBudgetStateHolder | None = None |