| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Sampling parameters for text generation.""" |
| 4 | |
| 5 | import copy |
| 6 | import json as json_mod |
| 7 | import math |
| 8 | from dataclasses import field |
| 9 | from enum import Enum, IntEnum |
| 10 | from functools import cached_property |
| 11 | from typing import Annotated, Any |
| 12 | |
| 13 | import msgspec |
| 14 | from pydantic import BeforeValidator |
| 15 | from pydantic.dataclasses import dataclass |
| 16 | |
| 17 | import vllm.envs as envs |
| 18 | from vllm.config import ModelConfig, SpeculativeConfig, StructuredOutputsConfig |
| 19 | from vllm.exceptions import VLLMValidationError |
| 20 | from vllm.logger import init_logger |
| 21 | from vllm.tokenizers import TokenizerLike |
| 22 | from vllm.utils.mistral import is_mistral_tokenizer |
| 23 | from vllm.v1.serial_utils import PydanticMsgspecMixin |
| 24 | |
| 25 | logger = init_logger(__name__) |
| 26 | |
| 27 | _SAMPLING_EPS = 1e-5 |
| 28 | _MAX_TEMP = 1e-2 |
| 29 | |
| 30 | MAX_LOGPROB_TOKEN_IDS = 128 |
| 31 | """Upper bound on `SamplingParams.logprob_token_ids` list length. Must match |
| 32 | the per-request row width allocated by the sampler's `LogprobTokenIdsState`.""" |
| 33 | |
| 34 | |
| 35 | def _verify_num_sequences(value: int, parameter_name: str) -> None: |
| 36 | if not isinstance(value, int): |
| 37 | raise VLLMValidationError( |
| 38 | f"{parameter_name} must be an int, but is of type {type(value)}" |
| 39 | ) |
| 40 | if value < 1: |
| 41 | raise VLLMValidationError(f"{parameter_name} must be at least 1, got {value}.") |
| 42 | max_n = envs.VLLM_MAX_N_SEQUENCES |
| 43 | if value > max_n: |
| 44 | raise VLLMValidationError( |
| 45 | f"{parameter_name} must be at most {max_n}, got {value}. " |
| 46 | "To increase this limit, set the VLLM_MAX_N_SEQUENCES " |
| 47 | "environment variable." |
| 48 | ) |
| 49 | |
| 50 | |
| 51 | def validate_thinking_token_budget(value: int | float | bool | None) -> int | None: |
| 52 | """Validate ``thinking_token_budget``; return ``None`` if unset.""" |
| 53 | if value is None: |
| 54 | return None |
| 55 | if isinstance(value, (bool, float)) or not isinstance(value, int): |
| 56 | raise VLLMValidationError( |
| 57 | "`thinking_token_budget` must be a non-negative integer " |
| 58 | "or -1 for unlimited.", |
| 59 | parameter="thinking_token_budget", |
| 60 | value=value, |
| 61 | ) |
| 62 | if value == -1: |
| 63 | return None |
| 64 | if value < 0: |
| 65 | raise VLLMValidationError( |
| 66 | "`thinking_token_budget` must be a non-negative integer " |
| 67 | "or -1 for unlimited.", |
| 68 | parameter="thinking_token_budget", |
| 69 | value=value, |
| 70 | ) |
| 71 | return value |
| 72 | |
| 73 | |
| 74 | ThinkingTokenBudget = Annotated[ |
| 75 | int | None, |
| 76 | BeforeValidator(validate_thinking_token_budget), |
| 77 | ] |
| 78 | |
| 79 | |
| 80 | class SamplingType(IntEnum): |
| 81 | GREEDY = 0 |
| 82 | RANDOM = 1 |
| 83 | RANDOM_SEED = 2 |
| 84 | |
| 85 | |
| 86 | # maybe make msgspec? |
| 87 | @dataclass |
| 88 | class StructuredOutputsParams: |
| 89 | # One of these fields will be used to build a logit processor. |
| 90 | json: str | dict | None = None |
| 91 | regex: str | None = None |
| 92 | choice: list[str] | None = None |
| 93 | grammar: str | None = None |
| 94 | json_object: bool | None = None |
| 95 | # These are other options that can be set. |
| 96 | disable_any_whitespace: bool = False |
| 97 | disable_additional_properties: bool = False |
| 98 | whitespace_pattern: str | None = None |
| 99 | structural_tag: str | None = None |
| 100 | |
| 101 | _backend: str | None = field(default=None, init=False) |
| 102 | """CAUTION: Should only be set by Processor._validate_structured_output""" |
| 103 | _backend_was_auto: bool = field(default=False, init=False) |
| 104 | """CAUTION: Should only be set by Processor._validate_structured_output""" |
| 105 | |
| 106 | def __post_init__(self): |
| 107 | """Validate that some fields are mutually exclusive.""" |
| 108 | count = sum( |
| 109 | [ |
| 110 | self.json is not None, |
| 111 | self.regex is not None, |
| 112 | self.choice is not None, |
| 113 | self.grammar is not None, |
| 114 | self.json_object is not None, |
| 115 | self.structural_tag is not None, |
| 116 | ] |
| 117 | ) |
| 118 | if count > 1: |
| 119 | raise VLLMValidationError( |
| 120 | "You can only use one kind of structured outputs constraint " |
| 121 | f"but multiple are specified: {self.__dict__}" |
| 122 | ) |
| 123 | if count < 1: |
| 124 | raise VLLMValidationError( |
| 125 | "You must use one kind of structured outputs constraint " |
| 126 | f"but none are specified: {self.__dict__}" |
| 127 | ) |
| 128 | |
| 129 | def all_constraints_none(self) -> bool: |
| 130 | """Returns True if all structured-output constraint fields are None.""" |
| 131 | return all( |
| 132 | getattr(self, field) is None |
| 133 | for field in ( |
| 134 | "json", |
| 135 | "regex", |
| 136 | "choice", |
| 137 | "grammar", |
| 138 | "json_object", |
| 139 | "structural_tag", |
| 140 | ) |
| 141 | ) |
| 142 | |
| 143 | def all_non_structural_tag_constraints_none(self) -> bool: |
| 144 | """Returns True if all structured-output constraint fields are None.""" |
| 145 | return all( |
| 146 | getattr(self, field) is None |
| 147 | for field in ( |
| 148 | "json", |
| 149 | "regex", |
| 150 | "choice", |
| 151 | "grammar", |
| 152 | "json_object", |
| 153 | ) |
| 154 | ) |
| 155 | |
| 156 | |
| 157 | @dataclass |
| 158 | class RepetitionDetectionParams: |
| 159 | """Parameters for detecting repetitive N-gram patterns in output tokens.""" |
| 160 | |
| 161 | max_pattern_size: int = 0 |
| 162 | """Maximum size of N-gram pattern to detect for sequence repetition. |
| 163 | Set to 0 to disable. Must be used together with min_count.""" |
| 164 | |
| 165 | min_pattern_size: int = 0 |
| 166 | """Minimum N-gram pattern size to check for sequence repetition. |
| 167 | If set to 0, it defaults to 1. |
| 168 | Must be <= max_pattern_size.""" |
| 169 | |
| 170 | min_count: int = 0 |
| 171 | """Minimum number of times an N-gram pattern must repeat to trigger |
| 172 | detection. Must be >= 2. Example: 3 for detecting a phrase repeated |
| 173 | 3 times. Must be used together with max_pattern_size.""" |
| 174 | |
| 175 | def __post_init__(self): |
| 176 | if ( |
| 177 | self.max_pattern_size < 0 |
| 178 | or self.min_pattern_size < 0 |
| 179 | or self.min_pattern_size > self.max_pattern_size |
| 180 | ): |
| 181 | raise VLLMValidationError( |
| 182 | "max_pattern_size, min_pattern_size must be >=0, " |
| 183 | "with min_pattern_size <= max_pattern_size. " |
| 184 | "Set both to 0 to disable repetitive pattern detection." |
| 185 | ) |
| 186 | if self.max_pattern_size > 0 and self.min_count < 2: |
| 187 | raise VLLMValidationError( |
| 188 | "min_count must be >= 2 to detect repetitive patterns " |
| 189 | "in engine output. If you do not wish to detect repetitive " |
| 190 | "patterns, set max_pattern_size to 0." |
| 191 | ) |
| 192 | |
| 193 | |
| 194 | class RequestOutputKind(Enum): |
| 195 | # Return entire output so far in every RequestOutput |
| 196 | CUMULATIVE = 0 |
| 197 | # Return only deltas in each RequestOutput |
| 198 | DELTA = 1 |
| 199 | # Do not return intermediate RequestOutput |
| 200 | FINAL_ONLY = 2 |
| 201 | |
| 202 | |
| 203 | def _is_non_tekken_mistral(tokenizer: TokenizerLike) -> bool: |
| 204 | return is_mistral_tokenizer(tokenizer) and not tokenizer.is_tekken |
| 205 | |
| 206 | |
| 207 | def _get_llg_tokenizer(tokenizer: TokenizerLike) -> Any: |
| 208 | return tokenizer.llg_tokenizer if is_mistral_tokenizer(tokenizer) else None |
| 209 | |
| 210 | |
| 211 | class SamplingParams( |
| 212 | PydanticMsgspecMixin, |
| 213 | msgspec.Struct, |
| 214 | omit_defaults=True, # type: ignore[call-arg] |
| 215 | # required for @cached_property. |
| 216 | dict=True, |
| 217 | ): # type: ignore[call-arg] |
| 218 | """Sampling parameters for text generation. |
| 219 | |
| 220 | Overall, we follow the sampling parameters from the OpenAI text completion |
| 221 | API (https://platform.openai.com/docs/api-reference/completions/create). |
| 222 | In addition, we support beam search, which is not supported by OpenAI. |
| 223 | """ |
| 224 | |
| 225 | n: int = 1 |
| 226 | """Number of outputs to return for the given prompt request. |
| 227 | |
| 228 | The maximum allowed value is controlled by the ``VLLM_MAX_N_SEQUENCES`` |
| 229 | environment variable (default: 16384). |
| 230 | |
| 231 | NOTE: |
| 232 | `AsyncLLM` streams outputs by default. When `n > 1`, all `n` outputs |
| 233 | are generated and streamed cumulatively per request. To see all `n` |
| 234 | outputs upon completion, use `output_kind=RequestOutputKind.FINAL_ONLY` |
| 235 | in `SamplingParams`.""" |
| 236 | presence_penalty: float = 0.0 |
| 237 | """Penalizes new tokens based on whether they appear in the generated text |
| 238 | so far. Values > 0 encourage the model to use new tokens, while values < 0 |
| 239 | encourage the model to repeat tokens.""" |
| 240 | frequency_penalty: float = 0.0 |
| 241 | """Penalizes new tokens based on their frequency in the generated text so |
| 242 | far. Values > 0 encourage the model to use new tokens, while values < 0 |
| 243 | encourage the model to repeat tokens.""" |
| 244 | repetition_penalty: float = 1.0 |
| 245 | """Penalizes new tokens based on whether they appear in the prompt and the |
| 246 | generated text so far. Values > 1 encourage the model to use new tokens, |
| 247 | while values < 1 encourage the model to repeat tokens.""" |
| 248 | temperature: float = 1.0 |
| 249 | """Controls the randomness of the sampling. Lower values make the model |
| 250 | more deterministic, while higher values make the model more random. Zero |
| 251 | means greedy sampling.""" |
| 252 | watermarking: bool = True |
| 253 | """Whether to apply the engine's configured watermark to this request.""" |
| 254 | top_p: float = 1.0 |
| 255 | """Controls the cumulative probability of the top tokens to consider. Must |
| 256 | be in (0, 1]. Set to 1 to consider all tokens.""" |
| 257 | top_k: int = 0 |
| 258 | """Controls the number of top tokens to consider. Set to 0 (or -1) to |
| 259 | consider all tokens.""" |
| 260 | min_p: float = 0.0 |
| 261 | """Represents the minimum probability for a token to be considered, |
| 262 | relative to the probability of the most likely token. Must be in [0, 1]. |
| 263 | Set to 0 to disable this.""" |
| 264 | seed: int | None = None |
| 265 | """Random seed to use for the generation.""" |
| 266 | stop: str | list[str] | None = None |
| 267 | """String(s) that stop the generation when they are generated. The returned |
| 268 | output will not contain the stop strings.""" |
| 269 | stop_token_ids: list[int] | None = None |
| 270 | """Token IDs that stop the generation when they are generated. The returned |
| 271 | output will contain the stop tokens unless the stop tokens are special |
| 272 | tokens.""" |
| 273 | ignore_eos: bool = False |
| 274 | """Whether to ignore the EOS token and continue generating |
| 275 | tokens after the EOS token is generated.""" |
| 276 | max_tokens: int | None = 16 |
| 277 | """Maximum number of tokens to generate per output sequence.""" |
| 278 | min_tokens: int = 0 |
| 279 | """Minimum number of tokens to generate per output sequence before EOS or |
| 280 | `stop_token_ids` can be generated""" |
| 281 | logprobs: int | None = None |
| 282 | """Number of log probabilities to return per output token. When set to |
| 283 | `None`, no probability is returned. If set to a non-`None` value, the |
| 284 | result includes the log probabilities of the specified number of most |
| 285 | likely tokens, as well as the chosen tokens. Note that the implementation |
| 286 | follows the OpenAI API: The API will always return the log probability of |
| 287 | the sampled token, so there may be up to `logprobs+1` elements in the |
| 288 | response. When set to -1, return all `vocab_size` log probabilities.""" |
| 289 | prompt_logprobs: int | None = None |
| 290 | """Number of log probabilities to return per prompt token. |
| 291 | When set to -1, return all `vocab_size` log probabilities.""" |
| 292 | prompt_logprob_token_ids: list[int] | None = None |
| 293 | """Token IDs to score at each scored causal prompt row, where row i scores |
| 294 | them as predictions of prompt token i + 1. The last prompt row is excluded, |
| 295 | so the result has `max(prompt_len - 1 - prompt_logprob_start, 0)` rows.""" |
| 296 | prompt_logprob_start: int | None = None |
| 297 | """First causal prompt row to score; defaults to the first row.""" |
| 298 | logprob_token_ids: list[int] | None = None |
| 299 | """Specific token IDs to return logprobs for. More efficient than |
| 300 | logprobs=-1 when you only need logprobs for a small set of tokens. |
| 301 | When set, logprobs for exactly these token IDs will be returned, |
| 302 | in addition to the sampled token. This is useful for scoring tasks |
| 303 | where you want to compare probabilities of specific label tokens.""" |
| 304 | flat_logprobs: bool = False |
| 305 | """Whether to return logprobs in flatten format (i.e. FlatLogprob) |
| 306 | for better performance. |
| 307 | NOTE: GC costs of FlatLogprobs is significantly smaller than |
| 308 | list[dict[int, Logprob]]. After enabled, PromptLogprobs and |
| 309 | SampleLogprobs would populated as FlatLogprobs.""" |
| 310 | # NOTE: This parameter is only exposed at the engine level for now. |
| 311 | # It is not exposed in the OpenAI API server, as the OpenAI API does |
| 312 | # not support returning only a list of token IDs. |
| 313 | detokenize: bool = True |
| 314 | """Whether to detokenize the output.""" |
| 315 | skip_special_tokens: bool = True |
| 316 | """Whether to skip special tokens in the output.""" |
| 317 | spaces_between_special_tokens: bool = True |
| 318 | """Whether to add spaces between special tokens in the output.""" |
| 319 | include_stop_str_in_output: bool = False |
| 320 | """Whether to include the stop strings in output text.""" |
| 321 | output_kind: RequestOutputKind = RequestOutputKind.CUMULATIVE |
| 322 | stream_interval: int | None = None |
| 323 | """Number of newly generated tokens to batch into each streamed |
| 324 | `RequestOutput`. Raises the interval above the engine-level |
| 325 | `--stream-interval`. Values below engine setting are clamped up to it. |
| 326 | The first and final outputs are always emitted immediately.""" |
| 327 | skip_clone: bool = False |
| 328 | """Internal flag indicating that this SamplingParams instance is safe to |
| 329 | reuse without cloning. When True, clone() will return self without |
| 330 | performing a deep copy. This should only be set when the params object |
| 331 | is guaranteed to be dedicated to a single request and won't be modified |
| 332 | in ways that would affect other uses.""" |
| 333 | |
| 334 | # The below fields are not supposed to be used as an input. |
| 335 | # They are set in post_init. |
| 336 | output_text_buffer_length: int = 0 |
| 337 | _eos_token_id: int | None = None |
| 338 | _all_stop_token_ids: set[int] = msgspec.field(default_factory=set) |
| 339 | |
| 340 | # Fields used to construct logits processors |
| 341 | structured_outputs: StructuredOutputsParams | None = None |
| 342 | """Parameters for configuring structured outputs.""" |
| 343 | logit_bias: dict[int, float] | None = None |
| 344 | """If provided, the engine will construct a logits processor that applies |
| 345 | these logit biases.""" |
| 346 | allowed_token_ids: list[int] | None = None |
| 347 | """If provided, the engine will construct a logits processor which only |
| 348 | retains scores for the given token ids.""" |
| 349 | extra_args: dict[str, Any] | None = None |
| 350 | """Arbitrary additional args, that can be used by custom sampling |
| 351 | implementations, plugins, etc. Not used by any in-tree sampling |
| 352 | implementations.""" |
| 353 | # Fields used for bad words |
| 354 | bad_words: list[str] | None = None |
| 355 | """Words that are not allowed to be generated. More precisely, only the |
| 356 | last token of a corresponding token sequence is not allowed when the next |
| 357 | generated token can complete the sequence.""" |
| 358 | _bad_words_token_ids: list[list[int]] | None = None |
| 359 | |
| 360 | skip_reading_prefix_cache: bool | None = None |
| 361 | thinking_token_budget: int | None = None |
| 362 | """Maximum number of tokens allowed for thinking operations.""" |
| 363 | |
| 364 | repetition_detection: RepetitionDetectionParams | None = None |
| 365 | """Parameters for detecting repetitive N-gram patterns in output tokens. |
| 366 | If such repetition is detected, generation will be ended early. LLMs can |
| 367 | sometimes generate repetitive, unhelpful token patterns, stopping only |
| 368 | when they hit the maximum output length (e.g. 'abcdabcdabcd...' or |
| 369 | '\\emoji \\emoji \\emoji ...'). This feature can detect such behavior |
| 370 | and terminate early, saving time and tokens.""" |
| 371 | |
| 372 | # Debugging / RL-specific parameters. Not intended for production serving. |
| 373 | routed_experts_prompt_start: int = 0 |
| 374 | """When enable_return_routed_experts is active, skip the first |
| 375 | routed_experts_prompt_start prompt tokens from the returned routing |
| 376 | data. In multi-turn agent scenarios, set this to the length of the |
| 377 | already-returned prefix to avoid duplicating routing for prompt tokens |
| 378 | covered by earlier turns. Default 0 returns routing for all prompt |
| 379 | tokens.""" |
| 380 | trace_decode_token_ids: list[int] | None = None |
| 381 | """If provided, forces the engine to emit this predetermined sequence of |
| 382 | token IDs during decoding instead of sampling randomly. Real logprobs are |
| 383 | still computed. Conflict checking is performed at the engine level.""" |
| 384 | |
| 385 | @staticmethod |
| 386 | def from_optional( |
| 387 | n: int | None = 1, |
| 388 | presence_penalty: float | None = 0.0, |
| 389 | frequency_penalty: float | None = 0.0, |
| 390 | repetition_penalty: float | None = 1.0, |
| 391 | temperature: float | None = 1.0, |
| 392 | watermarking: bool = True, |
| 393 | top_p: float | None = 1.0, |
| 394 | top_k: int = 0, |
| 395 | min_p: float = 0.0, |
| 396 | seed: int | None = None, |
| 397 | stop: str | list[str] | None = None, |
| 398 | stop_token_ids: list[int] | None = None, |
| 399 | bad_words: list[str] | None = None, |
| 400 | thinking_token_budget: int | None = None, |
| 401 | include_stop_str_in_output: bool = False, |
| 402 | ignore_eos: bool = False, |
| 403 | max_tokens: int | None = 16, |
| 404 | min_tokens: int = 0, |
| 405 | logprobs: int | None = None, |
| 406 | prompt_logprobs: int | None = None, |
| 407 | prompt_logprob_token_ids: list[int] | None = None, |
| 408 | prompt_logprob_start: int | None = None, |
| 409 | detokenize: bool = True, |
| 410 | skip_special_tokens: bool = True, |
| 411 | spaces_between_special_tokens: bool = True, |
| 412 | output_kind: RequestOutputKind = RequestOutputKind.CUMULATIVE, |
| 413 | stream_interval: int | None = None, |
| 414 | structured_outputs: StructuredOutputsParams | None = None, |
| 415 | logit_bias: dict[int, float] | dict[str, float] | None = None, |
| 416 | allowed_token_ids: list[int] | None = None, |
| 417 | extra_args: dict[str, Any] | None = None, |
| 418 | skip_clone: bool = False, |
| 419 | repetition_detection: RepetitionDetectionParams | None = None, |
| 420 | logprob_token_ids: list[int] | None = None, |
| 421 | routed_experts_prompt_start: int = 0, |
| 422 | # Debugging / RL-specific parameters. |
| 423 | trace_decode_token_ids: list[int] | None = None, |
| 424 | ) -> "SamplingParams": |
| 425 | if logit_bias is not None: |
| 426 | # Fast path uses a dict comprehension; on failure we iterate once |
| 427 | # to identify the exact offending entry for the error message. |
| 428 | try: |
| 429 | logit_bias = { |
| 430 | int(token): min(100.0, max(-100.0, bias)) |
| 431 | for token, bias in logit_bias.items() |
| 432 | } |
| 433 | except (ValueError, TypeError): |
| 434 | invalid_keys = [] |
| 435 | converted_logit_bias = {} |
| 436 | for token, bias in logit_bias.items(): |
| 437 | try: |
| 438 | token_id = int(token) |
| 439 | except (ValueError, TypeError): |
| 440 | invalid_keys.append(token) |
| 441 | continue |
| 442 | converted_logit_bias[token_id] = min(100.0, max(-100.0, bias)) |
| 443 | if invalid_keys: |
| 444 | raise VLLMValidationError( |
| 445 | f"logit_bias contains key(s) that cannot be " |
| 446 | f"converted to integer token IDs: {invalid_keys!r}", |
| 447 | parameter="logit_bias", |
| 448 | value=invalid_keys, |
| 449 | ) from None |
| 450 | logit_bias = converted_logit_bias |
| 451 | |
| 452 | return SamplingParams( |
| 453 | n=1 if n is None else n, |
| 454 | presence_penalty=0.0 if presence_penalty is None else presence_penalty, |
| 455 | frequency_penalty=0.0 if frequency_penalty is None else frequency_penalty, |
| 456 | repetition_penalty=1.0 |
| 457 | if repetition_penalty is None |
| 458 | else repetition_penalty, |
| 459 | temperature=1.0 if temperature is None else temperature, |
| 460 | watermarking=watermarking, |
| 461 | top_p=1.0 if top_p is None else top_p, |
| 462 | top_k=top_k, |
| 463 | min_p=min_p, |
| 464 | seed=seed, |
| 465 | stop=stop, |
| 466 | stop_token_ids=stop_token_ids, |
| 467 | bad_words=bad_words, |
| 468 | thinking_token_budget=thinking_token_budget, |
| 469 | include_stop_str_in_output=include_stop_str_in_output, |
| 470 | ignore_eos=ignore_eos, |
| 471 | max_tokens=max_tokens, |
| 472 | min_tokens=min_tokens, |
| 473 | logprobs=logprobs, |
| 474 | prompt_logprobs=prompt_logprobs, |
| 475 | prompt_logprob_token_ids=prompt_logprob_token_ids, |
| 476 | prompt_logprob_start=prompt_logprob_start, |
| 477 | logprob_token_ids=logprob_token_ids, |
| 478 | detokenize=detokenize, |
| 479 | skip_special_tokens=skip_special_tokens, |
| 480 | spaces_between_special_tokens=spaces_between_special_tokens, |
| 481 | output_kind=output_kind, |
| 482 | stream_interval=stream_interval, |
| 483 | structured_outputs=structured_outputs, |
| 484 | logit_bias=logit_bias, |
| 485 | allowed_token_ids=allowed_token_ids, |
| 486 | extra_args=extra_args, |
| 487 | skip_clone=skip_clone, |
| 488 | repetition_detection=repetition_detection, |
| 489 | routed_experts_prompt_start=routed_experts_prompt_start, |
| 490 | trace_decode_token_ids=trace_decode_token_ids, |
| 491 | ) |
| 492 | |
| 493 | def __post_init__(self) -> None: |
| 494 | if 0 < self.temperature < _MAX_TEMP: |
| 495 | logger.warning( |
| 496 | "temperature %s is less than %s, which may cause numerical " |
| 497 | "errors nan or inf in tensors. We have maxed it out to %s.", |
| 498 | self.temperature, |
| 499 | _MAX_TEMP, |
| 500 | _MAX_TEMP, |
| 501 | ) |
| 502 | self.temperature = max(self.temperature, _MAX_TEMP) |
| 503 | |
| 504 | if self.seed == -1: |
| 505 | self.seed = None |
| 506 | |
| 507 | self.thinking_token_budget = validate_thinking_token_budget( |
| 508 | self.thinking_token_budget |
| 509 | ) |
| 510 | |
| 511 | if self.stop is None: |
| 512 | self.stop = [] |
| 513 | elif isinstance(self.stop, str): |
| 514 | self.stop = [self.stop] |
| 515 | |
| 516 | if self.stop_token_ids is None: |
| 517 | self.stop_token_ids = [] |
| 518 | else: |
| 519 | self.stop_token_ids = list(dict.fromkeys(self.stop_token_ids)) |
| 520 | |
| 521 | if self.bad_words is None: |
| 522 | self.bad_words = [] |
| 523 | else: |
| 524 | self.bad_words = list(dict.fromkeys(self.bad_words)) |
| 525 | |
| 526 | if self.logprobs is True: |
| 527 | self.logprobs = 1 |
| 528 | |
| 529 | if self.prompt_logprobs is True: |
| 530 | self.prompt_logprobs = 1 |
| 531 | |
| 532 | # Number of characters to hold back for stop string evaluation |
| 533 | # until sequence is finished. |
| 534 | if self.stop and not self.include_stop_str_in_output: |
| 535 | self.output_text_buffer_length = max(len(s) for s in self.stop) - 1 |
| 536 | |
| 537 | self._verify_args() |
| 538 | |
| 539 | if ( |
| 540 | envs.VLLM_BATCH_INVARIANT |
| 541 | and self.temperature >= _SAMPLING_EPS |
| 542 | and self.seed is None |
| 543 | ): |
| 544 | logger.warning_once( |
| 545 | "Random sampling without an explicit seed may not be batch " |
| 546 | "invariant. Set seed in SamplingParams or use temperature=0." |
| 547 | ) |
| 548 | |
| 549 | if self.temperature < _SAMPLING_EPS: |
| 550 | # Zero temperature means greedy sampling. |
| 551 | self.top_p = 1.0 |
| 552 | self.top_k = 0 |
| 553 | self.min_p = 0.0 |
| 554 | self._verify_greedy_sampling() |
| 555 | |
| 556 | # eos_token_id is added to this by the engine |
| 557 | self._all_stop_token_ids.update(self.stop_token_ids) |
| 558 | |
| 559 | if self.skip_reading_prefix_cache is None: |
| 560 | # If prefix caching is enabled, |
| 561 | # the output of prompt logprobs may less than n_prompt_tokens, |
| 562 | # we need to skip reading cache at this request. |
| 563 | self.skip_reading_prefix_cache = ( |
| 564 | self.prompt_logprobs is not None |
| 565 | or self.prompt_logprob_token_ids is not None |
| 566 | ) |
| 567 | |
| 568 | def _verify_args(self) -> None: |
| 569 | _verify_num_sequences(self.n, "n") |
| 570 | if self.extra_args: |
| 571 | self._verify_extra_args() |
| 572 | if not -2.0 <= self.presence_penalty <= 2.0: |
| 573 | raise VLLMValidationError( |
| 574 | f"presence_penalty must be in [-2, 2], got {self.presence_penalty}." |
| 575 | ) |
| 576 | if not -2.0 <= self.frequency_penalty <= 2.0: |
| 577 | raise VLLMValidationError( |
| 578 | f"frequency_penalty must be in [-2, 2], got {self.frequency_penalty}." |
| 579 | ) |
| 580 | if not math.isfinite(self.repetition_penalty): |
| 581 | raise VLLMValidationError( |
| 582 | "repetition_penalty must be a finite number, " |
| 583 | f"got {self.repetition_penalty}." |
| 584 | ) |
| 585 | if self.repetition_penalty <= 0.0: |
| 586 | raise VLLMValidationError( |
| 587 | "repetition_penalty must be greater than zero, got " |
| 588 | f"{self.repetition_penalty}." |
| 589 | ) |
| 590 | if not math.isfinite(self.temperature): |
| 591 | raise VLLMValidationError( |
| 592 | f"temperature must be a finite number, got {self.temperature}.", |
| 593 | parameter="temperature", |
| 594 | value=self.temperature, |
| 595 | ) |
| 596 | if self.temperature < 0.0: |
| 597 | raise VLLMValidationError( |
| 598 | f"temperature must be non-negative, got {self.temperature}.", |
| 599 | parameter="temperature", |
| 600 | value=self.temperature, |
| 601 | ) |
| 602 | if self.temperature > 2.0: |
| 603 | raise VLLMValidationError( |
| 604 | f"temperature must be in [0, 2], got {self.temperature}.", |
| 605 | parameter="temperature", |
| 606 | value=self.temperature, |
| 607 | ) |
| 608 | if not 0.0 < self.top_p <= 1.0: |
| 609 | raise VLLMValidationError( |
| 610 | f"top_p must be in (0, 1], got {self.top_p}.", |
| 611 | parameter="top_p", |
| 612 | value=self.top_p, |
| 613 | ) |
| 614 | # quietly accept -1 as disabled, but prefer 0 |
| 615 | if self.top_k < -1: |
| 616 | raise VLLMValidationError( |
| 617 | f"top_k must be 0 (disable), or at least 1, got {self.top_k}." |
| 618 | ) |
| 619 | if not isinstance(self.top_k, int): |
| 620 | raise VLLMValidationError( |
| 621 | f"top_k must be an integer, got {type(self.top_k).__name__}" |
| 622 | ) |
| 623 | if not 0.0 <= self.min_p <= 1.0: |
| 624 | raise VLLMValidationError(f"min_p must be in [0, 1], got {self.min_p}.") |
| 625 | if self.max_tokens is not None and self.max_tokens < 1: |
| 626 | raise VLLMValidationError( |
| 627 | f"max_tokens must be at least 1, got {self.max_tokens}.", |
| 628 | parameter="max_tokens", |
| 629 | value=self.max_tokens, |
| 630 | ) |
| 631 | if self.min_tokens < 0: |
| 632 | raise VLLMValidationError( |
| 633 | f"min_tokens must be greater than or equal to 0, got {self.min_tokens}." |
| 634 | ) |
| 635 | if self.max_tokens is not None and self.min_tokens > self.max_tokens: |
| 636 | raise VLLMValidationError( |
| 637 | f"min_tokens must be less than or equal to " |
| 638 | f"max_tokens={self.max_tokens}, got {self.min_tokens}." |
| 639 | ) |
| 640 | if self.stream_interval is not None and self.stream_interval < 1: |
| 641 | raise VLLMValidationError( |
| 642 | f"stream_interval must be at least 1, got {self.stream_interval}.", |
| 643 | parameter="stream_interval", |
| 644 | value=self.stream_interval, |
| 645 | ) |
| 646 | if self.logprobs is not None and self.logprobs != -1 and self.logprobs < 0: |
| 647 | raise VLLMValidationError( |
| 648 | f"logprobs must be non-negative or -1, got {self.logprobs}.", |
| 649 | parameter="logprobs", |
| 650 | value=self.logprobs, |
| 651 | ) |
| 652 | if ( |
| 653 | self.prompt_logprobs is not None |
| 654 | and self.prompt_logprobs != -1 |
| 655 | and self.prompt_logprobs < 0 |
| 656 | ): |
| 657 | raise VLLMValidationError( |
| 658 | f"prompt_logprobs must be non-negative or -1, got " |
| 659 | f"{self.prompt_logprobs}.", |
| 660 | parameter="prompt_logprobs", |
| 661 | value=self.prompt_logprobs, |
| 662 | ) |
| 663 | assert isinstance(self.stop_token_ids, list) |
| 664 | if not all(isinstance(st_id, int) for st_id in self.stop_token_ids): |
| 665 | raise VLLMValidationError( |
| 666 | f"stop_token_ids must contain only integers, got {self.stop_token_ids}." |
| 667 | ) |
| 668 | assert isinstance(self.stop, list) |
| 669 | if any(not stop_str for stop_str in self.stop): |
| 670 | raise VLLMValidationError("stop cannot contain an empty string.") |
| 671 | if self.stop and not self.detokenize: |
| 672 | raise VLLMValidationError( |
| 673 | "stop strings are only supported when detokenize is True. " |
| 674 | "Set detokenize=True to use stop." |
| 675 | ) |
| 676 | assert isinstance(self.bad_words, list) |
| 677 | if any(not bad_word for bad_word in self.bad_words): |
| 678 | raise VLLMValidationError( |
| 679 | f"bad_words cannot contain an empty string. " |
| 680 | f"Got bad_words={self.bad_words}" |
| 681 | ) |
| 682 | |
| 683 | def _verify_extra_args(self) -> None: |
| 684 | # JSON accepts arbitrary integers, but the engine's MessagePack |
| 685 | # transport only supports signed/unsigned 64-bit integers. |
| 686 | pending: list[Any] = [self.extra_args] |
| 687 | visited: set[int] = set() |
| 688 | while pending: |
| 689 | value = pending.pop() |
| 690 | if isinstance(value, int) and not -(2**63) <= value < 2**64: |
| 691 | raise VLLMValidationError( |
| 692 | "extra_args integers must be between -2**63 and 2**64 - 1.", |
| 693 | parameter="extra_args", |
| 694 | ) |
| 695 | if isinstance(value, (dict, list, tuple)) and id(value) not in visited: |
| 696 | visited.add(id(value)) |
| 697 | if isinstance(value, dict): |
| 698 | pending.extend(value.keys()) |
| 699 | pending.extend(value.values()) |
| 700 | else: |
| 701 | pending.extend(value) |
| 702 | |
| 703 | def _verify_greedy_sampling(self) -> None: |
| 704 | if self.n > 1: |
| 705 | raise VLLMValidationError( |
| 706 | f"n must be 1 when using greedy sampling, got {self.n}." |
| 707 | ) |
| 708 | |
| 709 | def update_from_generation_config( |
| 710 | self, |
| 711 | generation_config: dict[str, Any], |
| 712 | eos_token_id: int | None = None, |
| 713 | ) -> None: |
| 714 | """Update if there are non-default values from generation_config.""" |
| 715 | if not self.ignore_eos: |
| 716 | self._eos_token_id = eos_token_id |
| 717 | |
| 718 | if eos_token_id is not None: |
| 719 | # Add the eos token id into the sampling_params to support |
| 720 | # min_tokens processing. |
| 721 | self._all_stop_token_ids.add(eos_token_id) |
| 722 | |
| 723 | # Update eos_token_id for generation |
| 724 | if (eos_ids := generation_config.get("eos_token_id")) is not None: |
| 725 | # it can be either int or list of int |
| 726 | eos_ids = {eos_ids} if isinstance(eos_ids, int) else set(eos_ids) |
| 727 | if eos_token_id is not None: |
| 728 | # We don't need to include the primary eos_token_id in |
| 729 | # stop_token_ids since it's handled separately for stopping |
| 730 | # purposes. |
| 731 | eos_ids.discard(eos_token_id) |
| 732 | if eos_ids: |
| 733 | self._all_stop_token_ids.update(eos_ids) |
| 734 | if not self.ignore_eos: |
| 735 | assert self.stop_token_ids is not None |
| 736 | eos_ids.update(self.stop_token_ids) |
| 737 | self.stop_token_ids = list(eos_ids) |
| 738 | |
| 739 | def update_from_tokenizer(self, tokenizer: TokenizerLike) -> None: |
| 740 | if not self.bad_words: |
| 741 | return |
| 742 | self._bad_words_token_ids = [] |
| 743 | max_num_bad_words = envs.VLLM_MAX_NUM_BAD_WORDS |
| 744 | for bad_word in self.bad_words: |
| 745 | # To prohibit words both at the beginning |
| 746 | # and in the middle of text |
| 747 | # (related to add_prefix_space tokenizer parameter) |
| 748 | for add_prefix_space in [False, True]: |
| 749 | prefix = " " if add_prefix_space else "" |
| 750 | prompt = prefix + bad_word.lstrip() |
| 751 | prompt_token_ids = tokenizer.encode( |
| 752 | text=prompt, add_special_tokens=False |
| 753 | ) |
| 754 | |
| 755 | if not prompt_token_ids: |
| 756 | if not add_prefix_space: |
| 757 | raise VLLMValidationError( |
| 758 | "bad_words entries must tokenize to at least one token.", |
| 759 | parameter="bad_words", |
| 760 | value=self.bad_words, |
| 761 | ) |
| 762 | # The unprefixed form is still enforceable when only the |
| 763 | # optional space-prefixed form tokenizes to nothing. |
| 764 | continue |
| 765 | |
| 766 | # If no space at the beginning |
| 767 | # or if prefix space produces a new word token |
| 768 | if (not add_prefix_space) or ( |
| 769 | add_prefix_space |
| 770 | and prompt_token_ids[0] != self._bad_words_token_ids[-1][0] |
| 771 | and len(prompt_token_ids) == len(self._bad_words_token_ids[-1]) |
| 772 | ): |
| 773 | self._bad_words_token_ids.append(prompt_token_ids) |
| 774 | if len(self._bad_words_token_ids) > max_num_bad_words: |
| 775 | raise VLLMValidationError( |
| 776 | f"Too many bad words after tokenization: " |
| 777 | f"{len(self._bad_words_token_ids)}. " |
| 778 | f"The max number is {max_num_bad_words}.", |
| 779 | parameter="bad_words", |
| 780 | value=self.bad_words, |
| 781 | ) |
| 782 | |
| 783 | invalid_token_ids = [ |
| 784 | token_id |
| 785 | for bad_words_token_ids in self._bad_words_token_ids |
| 786 | for token_id in bad_words_token_ids |
| 787 | if token_id < 0 or token_id > tokenizer.max_token_id |
| 788 | ] |
| 789 | if len(invalid_token_ids) > 0: |
| 790 | raise VLLMValidationError( |
| 791 | f"The model vocabulary size is {tokenizer.max_token_id + 1}," |
| 792 | f" but the following tokens" |
| 793 | f" were specified as bad: {invalid_token_ids}." |
| 794 | f" All token id values should be integers satisfying:" |
| 795 | f" 0 <= token_id <= {tokenizer.max_token_id}.", |
| 796 | parameter="bad_words", |
| 797 | value=self.bad_words, |
| 798 | ) |
| 799 | |
| 800 | @cached_property |
| 801 | def sampling_type(self) -> SamplingType: |
| 802 | if self.temperature < _SAMPLING_EPS: |
| 803 | return SamplingType.GREEDY |
| 804 | if self.seed is not None: |
| 805 | return SamplingType.RANDOM_SEED |
| 806 | return SamplingType.RANDOM |
| 807 | |
| 808 | @property |
| 809 | def eos_token_id(self) -> int | None: |
| 810 | return self._eos_token_id |
| 811 | |
| 812 | @property |
| 813 | def all_stop_token_ids(self) -> set[int]: |
| 814 | return self._all_stop_token_ids |
| 815 | |
| 816 | @property |
| 817 | def bad_words_token_ids(self) -> list[list[int]] | None: |
| 818 | # For internal use only. Backward compatibility not guaranteed |
| 819 | return self._bad_words_token_ids |
| 820 | |
| 821 | @property |
| 822 | def num_logprobs(self) -> int | None: |
| 823 | """Number of sample logprobs to return per output token, or `None` if |
| 824 | no sample logprobs were requested. Takes `logprob_token_ids` into |
| 825 | account: when `logprobs` is unset but `logprob_token_ids` is set, |
| 826 | returns `len(logprob_token_ids)`.""" |
| 827 | if self.logprobs is not None: |
| 828 | return self.logprobs |
| 829 | return len(self.logprob_token_ids) if self.logprob_token_ids else None |
| 830 | |
| 831 | def clone(self) -> "SamplingParams": |
| 832 | """If skip_clone is True, uses shallow copy instead of deep copy.""" |
| 833 | if self.skip_clone: |
| 834 | return copy.copy(self) |
| 835 | |
| 836 | return copy.deepcopy(self) |
| 837 | |
| 838 | def verify( |
| 839 | self, |
| 840 | model_config: ModelConfig, |
| 841 | speculative_config: SpeculativeConfig | None, |
| 842 | structured_outputs_config: StructuredOutputsConfig | None, |
| 843 | tokenizer: TokenizerLike | None, |
| 844 | ) -> None: |
| 845 | self._validate_logprobs(model_config) |
| 846 | self._validate_logit_bias(model_config) |
| 847 | self._validate_trace_replay(model_config, speculative_config) |
| 848 | self._validate_stop_token_ids(model_config) |
| 849 | self._validate_allowed_token_ids(model_config) |
| 850 | self._validate_spec_decode(speculative_config) |
| 851 | self._validate_diffusion(model_config) |
| 852 | self._validate_structured_outputs( |
| 853 | model_config, structured_outputs_config, tokenizer |
| 854 | ) |
| 855 | |
| 856 | def _validate_logprobs(self, model_config: ModelConfig) -> None: |
| 857 | max_logprobs = model_config.max_logprobs |
| 858 | if max_logprobs == -1: |
| 859 | max_logprobs = model_config.get_vocab_size() |
| 860 | |
| 861 | # Validate sample logprobs. |
| 862 | if num_logprobs := self.logprobs: |
| 863 | if num_logprobs == -1: |
| 864 | num_logprobs = model_config.get_vocab_size() |
| 865 | if num_logprobs > max_logprobs: |
| 866 | raise VLLMValidationError( |
| 867 | f"Requested sample logprobs of {num_logprobs}, " |
| 868 | f"which is greater than max allowed: {max_logprobs}", |
| 869 | parameter="logprobs", |
| 870 | value=num_logprobs, |
| 871 | ) |
| 872 | |
| 873 | # Validate logprob_token_ids. |
| 874 | if self.logprob_token_ids is not None: |
| 875 | n = len(self.logprob_token_ids) |
| 876 | if n > MAX_LOGPROB_TOKEN_IDS: |
| 877 | raise VLLMValidationError( |
| 878 | f"Requested logprob_token_ids of length {n}, " |
| 879 | f"which is greater than max allowed: {MAX_LOGPROB_TOKEN_IDS}", |
| 880 | parameter="logprob_token_ids", |
| 881 | value=n, |
| 882 | ) |
| 883 | vocab_size = model_config.get_vocab_size() |
| 884 | invalid_token_ids = [ |
| 885 | token_id |
| 886 | for token_id in self.logprob_token_ids |
| 887 | if token_id < 0 or token_id >= vocab_size |
| 888 | ] |
| 889 | if invalid_token_ids: |
| 890 | raise VLLMValidationError( |
| 891 | f"token_id(s) {invalid_token_ids} in logprob_token_ids " |
| 892 | f"contain out-of-vocab token ids. Vocabulary size: " |
| 893 | f"{vocab_size}", |
| 894 | parameter="logprob_token_ids", |
| 895 | value=invalid_token_ids, |
| 896 | ) |
| 897 | if self.logprobs is not None and self.logprobs != n: |
| 898 | raise VLLMValidationError( |
| 899 | f"When both logprobs and logprob_token_ids are set, " |
| 900 | f"logprobs must equal len(logprob_token_ids). Got " |
| 901 | f"logprobs={self.logprobs}, len(logprob_token_ids)={n}.", |
| 902 | parameter="logprob_token_ids", |
| 903 | value=n, |
| 904 | ) |
| 905 | |
| 906 | # Validate prompt logprobs. |
| 907 | if num_prompt_logprobs := self.prompt_logprobs: |
| 908 | if num_prompt_logprobs == -1: |
| 909 | num_prompt_logprobs = model_config.get_vocab_size() |
| 910 | if num_prompt_logprobs > max_logprobs: |
| 911 | raise VLLMValidationError( |
| 912 | f"Requested prompt logprobs of {num_prompt_logprobs}, " |
| 913 | f"which is greater than max allowed: {max_logprobs}", |
| 914 | parameter="prompt_logprobs", |
| 915 | value=num_prompt_logprobs, |
| 916 | ) |
| 917 | |
| 918 | # Validate prompt_logprob_token_ids. |
| 919 | if self.prompt_logprob_token_ids is not None: |
| 920 | n = len(self.prompt_logprob_token_ids) |
| 921 | if n == 0: |
| 922 | raise VLLMValidationError( |
| 923 | "prompt_logprob_token_ids must not be empty.", |
| 924 | parameter="prompt_logprob_token_ids", |
| 925 | value=n, |
| 926 | ) |
| 927 | if n > max_logprobs: |
| 928 | raise VLLMValidationError( |
| 929 | f"Requested prompt_logprob_token_ids of length {n}, " |
| 930 | f"which is greater than max allowed: {max_logprobs}. " |
| 931 | f"Set max_logprobs (--max-logprobs) to at least {n}.", |
| 932 | parameter="prompt_logprob_token_ids", |
| 933 | value=n, |
| 934 | ) |
| 935 | vocab_size = model_config.get_vocab_size() |
| 936 | invalid_token_ids = [ |
| 937 | token_id |
| 938 | for token_id in self.prompt_logprob_token_ids |
| 939 | if token_id < 0 or token_id >= vocab_size |
| 940 | ] |
| 941 | if invalid_token_ids: |
| 942 | raise VLLMValidationError( |
| 943 | f"token_id(s) {invalid_token_ids} in " |
| 944 | f"prompt_logprob_token_ids contain out-of-vocab token ids. " |
| 945 | f"Vocabulary size: {vocab_size}", |
| 946 | parameter="prompt_logprob_token_ids", |
| 947 | value=invalid_token_ids, |
| 948 | ) |
| 949 | if len(set(self.prompt_logprob_token_ids)) != n: |
| 950 | raise VLLMValidationError( |
| 951 | "prompt_logprob_token_ids must not contain duplicates.", |
| 952 | parameter="prompt_logprob_token_ids", |
| 953 | value=self.prompt_logprob_token_ids, |
| 954 | ) |
| 955 | if self.prompt_logprob_start is not None and self.prompt_logprob_start < 0: |
| 956 | raise VLLMValidationError( |
| 957 | "prompt_logprob_start must be non-negative.", |
| 958 | parameter="prompt_logprob_start", |
| 959 | value=self.prompt_logprob_start, |
| 960 | ) |
| 961 | elif self.prompt_logprob_start is not None: |
| 962 | raise VLLMValidationError( |
| 963 | "prompt_logprob_start requires prompt_logprob_token_ids.", |
| 964 | parameter="prompt_logprob_start", |
| 965 | value=self.prompt_logprob_start, |
| 966 | ) |
| 967 | |
| 968 | def _validate_stop_token_ids(self, model_config: ModelConfig) -> None: |
| 969 | """Validate stop_token_ids are within vocabulary range.""" |
| 970 | if not self.stop_token_ids: |
| 971 | return |
| 972 | |
| 973 | # stop_token_ids are used as column indices into the logits tensor, |
| 974 | # whose width is the model's vocab size (LogitsProcessor is built from |
| 975 | # config.vocab_size, InputBatch.vocab_size comes from |
| 976 | # model_config.get_vocab_size()), so use the same bound here — like |
| 977 | # _validate_logit_bias, which indexes the same tensor. |
| 978 | vocab_size = model_config.get_vocab_size() |
| 979 | invalid_token_ids = [ |
| 980 | token_id |
| 981 | for token_id in self.stop_token_ids |
| 982 | if token_id < 0 or token_id >= vocab_size |
| 983 | ] |
| 984 | |
| 985 | if invalid_token_ids: |
| 986 | raise VLLMValidationError( |
| 987 | f"token_id(s) {invalid_token_ids} in stop_token_ids contain " |
| 988 | f"out-of-vocab token ids. Vocabulary size: {vocab_size}", |
| 989 | parameter="stop_token_ids", |
| 990 | value=invalid_token_ids, |
| 991 | ) |
| 992 | |
| 993 | def _validate_logit_bias(self, model_config: ModelConfig) -> None: |
| 994 | """Validate logit_bias token IDs are within vocabulary range.""" |
| 995 | if not self.logit_bias: |
| 996 | return |
| 997 | |
| 998 | vocab_size = model_config.get_vocab_size() |
| 999 | invalid_token_ids = [ |
| 1000 | token_id |
| 1001 | for token_id in self.logit_bias |
| 1002 | if token_id < 0 or token_id >= vocab_size |
| 1003 | ] |
| 1004 | |
| 1005 | if invalid_token_ids: |
| 1006 | raise VLLMValidationError( |
| 1007 | f"token_id(s) {invalid_token_ids} in logit_bias contain " |
| 1008 | f"out-of-vocab token ids. Vocabulary size: {vocab_size}", |
| 1009 | parameter="logit_bias", |
| 1010 | value=invalid_token_ids, |
| 1011 | ) |
| 1012 | |
| 1013 | def _validate_trace_replay( |
| 1014 | self, |
| 1015 | model_config: ModelConfig, |
| 1016 | speculative_config: SpeculativeConfig | None, |
| 1017 | ) -> None: |
| 1018 | """Validate trace replay request compatibility.""" |
| 1019 | if self.trace_decode_token_ids is None: |
| 1020 | return |
| 1021 | |
| 1022 | if len(self.trace_decode_token_ids) == 0: |
| 1023 | raise ValueError("trace_decode_token_ids must be a non-empty list.") |
| 1024 | if self.n != 1: |
| 1025 | raise ValueError("trace_decode_token_ids requires n=1.") |
| 1026 | if not all(isinstance(t, int) and t >= 0 for t in self.trace_decode_token_ids): |
| 1027 | raise ValueError( |
| 1028 | "trace_decode_token_ids must contain non-negative integers." |
| 1029 | ) |
| 1030 | |
| 1031 | if self.prompt_logprobs is not None: |
| 1032 | raise ValueError( |
| 1033 | "trace_decode_token_ids is not supported with prompt_logprobs." |
| 1034 | ) |
| 1035 | if self.prompt_logprob_token_ids is not None: |
| 1036 | raise ValueError( |
| 1037 | "trace_decode_token_ids is not supported with prompt_logprob_token_ids." |
| 1038 | ) |
| 1039 | if speculative_config is not None: |
| 1040 | raise ValueError( |
| 1041 | "trace_decode_token_ids is not supported with speculative decoding." |
| 1042 | ) |
| 1043 | if self.structured_outputs is not None: |
| 1044 | raise ValueError( |
| 1045 | "trace_decode_token_ids is not supported with structured outputs." |
| 1046 | ) |
| 1047 | if self.repetition_detection is not None: |
| 1048 | raise ValueError( |
| 1049 | "trace_decode_token_ids is not supported with repetition_detection." |
| 1050 | ) |
| 1051 | if self.thinking_token_budget is not None: |
| 1052 | raise ValueError( |
| 1053 | "trace_decode_token_ids is not supported with thinking_token_budget." |
| 1054 | ) |
| 1055 | if self.bad_words: |
| 1056 | raise ValueError("trace_decode_token_ids is not supported with bad_words.") |
| 1057 | |
| 1058 | vocab_size = model_config.get_vocab_size() |
| 1059 | invalid_token_ids = [ |
| 1060 | token_id |
| 1061 | for token_id in self.trace_decode_token_ids |
| 1062 | if token_id < 0 or token_id >= vocab_size |
| 1063 | ] |
| 1064 | if invalid_token_ids: |
| 1065 | raise VLLMValidationError( |
| 1066 | f"token_id(s) {invalid_token_ids} in trace_decode_token_ids " |
| 1067 | f"contain out-of-vocab token ids. Vocabulary size: {vocab_size}", |
| 1068 | parameter="trace_decode_token_ids", |
| 1069 | value=invalid_token_ids, |
| 1070 | ) |
| 1071 | |
| 1072 | def _validate_allowed_token_ids(self, model_config: ModelConfig) -> None: |
| 1073 | allowed_token_ids = self.allowed_token_ids |
| 1074 | if allowed_token_ids is None: |
| 1075 | return |
| 1076 | |
| 1077 | if len(allowed_token_ids) == 0: |
| 1078 | raise VLLMValidationError( |
| 1079 | "allowed_token_ids is not None and empty!", |
| 1080 | parameter="allowed_token_ids", |
| 1081 | value=allowed_token_ids, |
| 1082 | ) |
| 1083 | |
| 1084 | # allowed_token_ids are client-supplied ids used as column indices |
| 1085 | # into the logits tensor (the mask in InputBatch is sized by |
| 1086 | # model_config.get_vocab_size(), and the ids are written as |
| 1087 | # mask[req_index][allowed_token_ids]), so use the same bound here — |
| 1088 | # like _validate_stop_token_ids and _validate_logit_bias, which |
| 1089 | # index the same tensor. |
| 1090 | vocab_size = model_config.get_vocab_size() |
| 1091 | invalid_token_ids = [ |
| 1092 | token_id |
| 1093 | for token_id in allowed_token_ids |
| 1094 | if token_id < 0 or token_id >= vocab_size |
| 1095 | ] |
| 1096 | if invalid_token_ids: |
| 1097 | raise VLLMValidationError( |
| 1098 | "allowed_token_ids contains out-of-vocab token id!", |
| 1099 | parameter="allowed_token_ids", |
| 1100 | value=invalid_token_ids, |
| 1101 | ) |
| 1102 | |
| 1103 | def _validate_spec_decode( |
| 1104 | self, |
| 1105 | speculative_config: SpeculativeConfig | None, |
| 1106 | ) -> None: |
| 1107 | if speculative_config is None: |
| 1108 | return |
| 1109 | |
| 1110 | # Some sampling parameters are not yet compatible with spec decoding. |
| 1111 | if self.min_p > _SAMPLING_EPS or self.logit_bias: |
| 1112 | raise VLLMValidationError( |
| 1113 | "The min_p and logit_bias sampling parameters " |
| 1114 | "are not yet supported with speculative decoding." |
| 1115 | ) |
| 1116 | |
| 1117 | def _validate_diffusion(self, model_config: ModelConfig) -> None: |
| 1118 | if not model_config.is_diffusion: |
| 1119 | return |
| 1120 | |
| 1121 | # Diffusion models denoise a whole canvas per step with a fixed |
| 1122 | # temperature schedule, so per-request sampling parameters are not |
| 1123 | # supported. Penalties are ignored by the sampler with a warning. |
| 1124 | if ( |
| 1125 | self.temperature != 1.0 |
| 1126 | or self.min_p > _SAMPLING_EPS |
| 1127 | or self.seed is not None |
| 1128 | or self.min_tokens > 0 |
| 1129 | or self.logit_bias |
| 1130 | or self.bad_words |
| 1131 | or self.allowed_token_ids |
| 1132 | ): |
| 1133 | raise VLLMValidationError( |
| 1134 | "The temperature, min_p, seed, min_tokens, logit_bias, " |
| 1135 | "bad_words, and allowed_token_ids sampling parameters " |
| 1136 | "are not yet supported with diffusion models." |
| 1137 | ) |
| 1138 | |
| 1139 | def _validate_structured_outputs( |
| 1140 | self, |
| 1141 | model_config: ModelConfig, |
| 1142 | structured_outputs_config: StructuredOutputsConfig | None, |
| 1143 | tokenizer: TokenizerLike | None, |
| 1144 | ) -> None: |
| 1145 | if structured_outputs_config is None or self.structured_outputs is None: |
| 1146 | return |
| 1147 | |
| 1148 | if model_config.is_diffusion: |
| 1149 | # Diffusion LLMs denoise a whole canvas of tokens in parallel |
| 1150 | # rather than sampling left-to-right, which the grammar FSM |
| 1151 | # requires. Without this check, requests fail mid-generation |
| 1152 | # with an FSM rejection (HTTP 500). See issue #45436. |
| 1153 | raise VLLMValidationError( |
| 1154 | "Structured outputs are not yet supported for diffusion " |
| 1155 | "language models. Remove the structured output constraint " |
| 1156 | "(e.g. `response_format`, `structured_outputs`) from the " |
| 1157 | "request." |
| 1158 | ) |
| 1159 | |
| 1160 | if tokenizer is None: |
| 1161 | raise VLLMValidationError( |
| 1162 | "Structured outputs requires a tokenizer so it can't be used with 'skip_tokenizer_init'" # noqa: E501 |
| 1163 | ) |
| 1164 | |
| 1165 | backend = structured_outputs_config.backend |
| 1166 | if _backend := self.structured_outputs._backend: |
| 1167 | # Request-level backend selection is not supported. |
| 1168 | # The values may differ if `params` is reused and was set |
| 1169 | # to a specific backend based on `auto` behavior in a previous |
| 1170 | # request. We remember that it was set as a result of `auto` |
| 1171 | # using the `_backend_was_auto` field set in the params. |
| 1172 | if backend != _backend and not ( |
| 1173 | backend == "auto" and self.structured_outputs._backend_was_auto |
| 1174 | ): |
| 1175 | raise VLLMValidationError( |
| 1176 | "Request-level structured output backend selection is not " |
| 1177 | f"supported. The request specified '{_backend}', but vLLM " |
| 1178 | f"was initialised with '{backend}'. This error can be " |
| 1179 | "resolved by removing '_backend' from the request." |
| 1180 | ) |
| 1181 | else: |
| 1182 | self.structured_outputs._backend = backend |
| 1183 | |
| 1184 | # Request content validation |
| 1185 | if ( |
| 1186 | isinstance(self.structured_outputs.choice, list) |
| 1187 | and not self.structured_outputs.choice |
| 1188 | ): |
| 1189 | # It is invalid for choice to be an empty list |
| 1190 | raise VLLMValidationError( |
| 1191 | f"Choice '{self.structured_outputs.choice}' cannot be an empty list" # noqa: E501 |
| 1192 | ) |
| 1193 | # Reject empty string grammar early to avoid engine-side crashes |
| 1194 | if ( |
| 1195 | isinstance(self.structured_outputs.grammar, str) |
| 1196 | and self.structured_outputs.grammar.strip() == "" |
| 1197 | ): |
| 1198 | raise VLLMValidationError( |
| 1199 | "structured_outputs.grammar cannot be an empty string" |
| 1200 | ) |
| 1201 | # Reject empty string json schema early to avoid engine-side crashes |
| 1202 | if ( |
| 1203 | isinstance(self.structured_outputs.json, str) |
| 1204 | and self.structured_outputs.json.strip() == "" |
| 1205 | ): |
| 1206 | raise VLLMValidationError( |
| 1207 | "structured_outputs.json cannot be an empty string" |
| 1208 | ) |
| 1209 | # Reject json_object=False early to avoid engine-side crashes |
| 1210 | if self.structured_outputs.json_object is False: |
| 1211 | raise VLLMValidationError( |
| 1212 | "structured_outputs.json_object must be True if set; omit " |
| 1213 | "structured_outputs to disable structured outputs" |
| 1214 | ) |
| 1215 | # Reject a regex containing a NUL byte early, in every backend mode. A |
| 1216 | # NUL is never meaningful in a regex pattern and is not handled by the |
| 1217 | # regex-to-grammar conversion. Checked here, before backend selection, |
| 1218 | # so it is a clean 400 rather than a silent fallback in the default |
| 1219 | # "auto" mode. |
| 1220 | if self.structured_outputs.regex and "\x00" in self.structured_outputs.regex: |
| 1221 | raise VLLMValidationError( |
| 1222 | "structured_outputs.regex must not contain a NUL character ('\\x00')" |
| 1223 | ) |
| 1224 | # Note(arpera): |
| 1225 | # We do NOT check here structured output regex on emptiness because |
| 1226 | # empty regex is indeed compiles to a valid grammar as well as |
| 1227 | # whitespace-only regexps, for instance, regex="\n" or regex=" " |
| 1228 | # are valid patterns and we MUST process them. |
| 1229 | if ( |
| 1230 | isinstance(self.structured_outputs.structural_tag, str) |
| 1231 | and self.structured_outputs.structural_tag.strip() == "" |
| 1232 | ): |
| 1233 | raise VLLMValidationError( |
| 1234 | "structured_outputs.structural_tag cannot be an empty string" |
| 1235 | ) |
| 1236 | |
| 1237 | from vllm.v1.structured_output.backend_guidance import ( |
| 1238 | has_guidance_unsupported_json_features, |
| 1239 | validate_guidance_grammar, |
| 1240 | ) |
| 1241 | from vllm.v1.structured_output.backend_lm_format_enforcer import ( |
| 1242 | validate_structured_output_request_lm_format_enforcer, |
| 1243 | ) |
| 1244 | from vllm.v1.structured_output.backend_outlines import ( |
| 1245 | validate_structured_output_request_outlines, |
| 1246 | ) |
| 1247 | from vllm.v1.structured_output.backend_xgrammar import validate_xgrammar_grammar |
| 1248 | from vllm.v1.structured_output.utils import grammar_is_likely_lark |
| 1249 | |
| 1250 | if backend.startswith("xgrammar"): |
| 1251 | # xgrammar with no fallback |
| 1252 | validate_xgrammar_grammar(self) |
| 1253 | elif backend.startswith("guidance"): |
| 1254 | if _is_non_tekken_mistral(tokenizer=tokenizer): |
| 1255 | raise VLLMValidationError( |
| 1256 | "Non-tekken Mistral tokenizers are not supported for the 'guidance'" |
| 1257 | " structured output backend. Please either use a more recent " |
| 1258 | "Mistral model, the ['xgrammar', 'outlines'] " |
| 1259 | "backends or tokenizer_mode='hf' instead." |
| 1260 | ) |
| 1261 | # TODO: ideally we would have the LLTokenizer here as Lark syntax |
| 1262 | # allows <|special_token|> and similar, see |
| 1263 | # https://github.com/guidance-ai/llguidance/blob/main/docs/syntax.md#special-tokens |
| 1264 | # Without tokenizer these are disallowed in grammars. |
| 1265 | validate_guidance_grammar( |
| 1266 | self, |
| 1267 | tokenizer=_get_llg_tokenizer(tokenizer), |
| 1268 | ) |
| 1269 | elif backend == "outlines": |
| 1270 | # outlines backend |
| 1271 | validate_structured_output_request_outlines(self) |
| 1272 | elif backend == "lm-format-enforcer": |
| 1273 | # lm format enforcer backend |
| 1274 | if is_mistral_tokenizer(tokenizer): |
| 1275 | raise VLLMValidationError( |
| 1276 | "Mistral tokenizer is not supported for the 'lm-format-enforcer' " |
| 1277 | "structured output backend. Please use ['xgrammar', 'outlines'] " |
| 1278 | "backends or tokenizer_mode='hf' instead." |
| 1279 | ) |
| 1280 | validate_structured_output_request_lm_format_enforcer(self) |
| 1281 | elif ( |
| 1282 | backend == "auto" |
| 1283 | and is_mistral_tokenizer(tokenizer) |
| 1284 | and tokenizer.is_tekken |
| 1285 | and self.structured_outputs.grammar |
| 1286 | and grammar_is_likely_lark(self.structured_outputs.grammar) |
| 1287 | ): |
| 1288 | # Lark grammars for Tekken Mistral tokenizers, including the ones |
| 1289 | # the Mistral tool parser generates, go to guidance. llguidance |
| 1290 | # takes the tokenizer from mistral-common and never lets a regex |
| 1291 | # match a special token. The xgrammar backend declares no special |
| 1292 | # tokens for Mistral tokenizers, so a regex like `.` can match |
| 1293 | # `[TOOL_CALLS]` or `[INST]` as plain text. |
| 1294 | validate_guidance_grammar(self, tokenizer=_get_llg_tokenizer(tokenizer)) |
| 1295 | self.structured_outputs._backend = "guidance" |
| 1296 | self.structured_outputs._backend_was_auto = True |
| 1297 | else: |
| 1298 | # NOTE: backend must be "auto" here, because we have |
| 1299 | # checked supported_backends above. |
| 1300 | # In this mode, we set opinionated defaults based on what we think |
| 1301 | # will satisfy the most use cases without having to worry about |
| 1302 | # this setting. We include fallback behavior here, but not with any |
| 1303 | # other setting where a specific backend was specified. |
| 1304 | try: |
| 1305 | validate_xgrammar_grammar(self) |
| 1306 | self.structured_outputs._backend = "xgrammar" |
| 1307 | except VLLMValidationError: |
| 1308 | # The request either failed validation |
| 1309 | # or includes some jsonschema feature(s) that |
| 1310 | # are not supported in xgrammar. |
| 1311 | |
| 1312 | skip_guidance = _is_non_tekken_mistral(tokenizer) |
| 1313 | |
| 1314 | # Check if schema has features unsupported by guidance |
| 1315 | so_params = self.structured_outputs |
| 1316 | if not skip_guidance and so_params.json: |
| 1317 | if isinstance(so_params.json, str): |
| 1318 | try: |
| 1319 | schema = json_mod.loads(so_params.json) |
| 1320 | except json_mod.JSONDecodeError as e: |
| 1321 | raise VLLMValidationError( |
| 1322 | "Invalid JSON grammar specification." |
| 1323 | ) from e |
| 1324 | else: |
| 1325 | schema = so_params.json |
| 1326 | skip_guidance = has_guidance_unsupported_json_features(schema) |
| 1327 | |
| 1328 | if skip_guidance: |
| 1329 | # Fall back to outlines if the tokenizer is non-tekken Mistral or |
| 1330 | # the schema contains features unsupported by guidance |
| 1331 | validate_structured_output_request_outlines(self) |
| 1332 | self.structured_outputs._backend = "outlines" |
| 1333 | else: |
| 1334 | # Fall back to guidance by default. |
| 1335 | validate_guidance_grammar( |
| 1336 | self, |
| 1337 | tokenizer=_get_llg_tokenizer(tokenizer), |
| 1338 | ) |
| 1339 | self.structured_outputs._backend = "guidance" |
| 1340 | # Remember that this backend was set automatically |
| 1341 | self.structured_outputs._backend_was_auto = True |
| 1342 | |
| 1343 | # Run post-init validation. This is also important to ensure subsequent |
| 1344 | # roundtrip serialization/deserialization won't fail. |
| 1345 | self.structured_outputs.__post_init__() |
| 1346 | |
| 1347 | def __repr__(self) -> str: |
| 1348 | return ( |
| 1349 | f"SamplingParams(n={self.n}, " |
| 1350 | f"presence_penalty={self.presence_penalty}, " |
| 1351 | f"frequency_penalty={self.frequency_penalty}, " |
| 1352 | f"repetition_penalty={self.repetition_penalty}, " |
| 1353 | f"temperature={self.temperature}, " |
| 1354 | f"watermarking={self.watermarking}, " |
| 1355 | f"top_p={self.top_p}, " |
| 1356 | f"top_k={self.top_k}, " |
| 1357 | f"min_p={self.min_p}, " |
| 1358 | f"seed={self.seed}, " |
| 1359 | f"stop={self.stop}, " |
| 1360 | f"stop_token_ids={self.stop_token_ids}, " |
| 1361 | f"bad_words={self.bad_words}, " |
| 1362 | f"thinking_token_budget={self.thinking_token_budget}, " |
| 1363 | f"include_stop_str_in_output={self.include_stop_str_in_output}, " |
| 1364 | f"ignore_eos={self.ignore_eos}, " |
| 1365 | f"max_tokens={self.max_tokens}, " |
| 1366 | f"min_tokens={self.min_tokens}, " |
| 1367 | f"logprobs={self.logprobs}, " |
| 1368 | f"prompt_logprobs={self.prompt_logprobs}, " |
| 1369 | f"skip_special_tokens={self.skip_special_tokens}, " |
| 1370 | "spaces_between_special_tokens=" |
| 1371 | f"{self.spaces_between_special_tokens}, " |
| 1372 | f"structured_outputs={self.structured_outputs}, " |
| 1373 | f"extra_args={self.extra_args})" |
| 1374 | ) |
| 1375 | |
| 1376 | @staticmethod |
| 1377 | def for_sampler_warmup() -> "SamplingParams": |
| 1378 | """Set parameters to exercise all sampler logic.""" |
| 1379 | return SamplingParams( |
| 1380 | temperature=0.9, |
| 1381 | top_p=0.9, |
| 1382 | top_k=50, |
| 1383 | min_p=0.1, |
| 1384 | frequency_penalty=0.5, |
| 1385 | presence_penalty=0.5, |
| 1386 | repetition_penalty=1.2, |
| 1387 | min_tokens=2, |
| 1388 | logit_bias={0: -1.0, 1: 0.5}, |
| 1389 | _bad_words_token_ids=[[0], [1, 2]], |
| 1390 | logprobs=5, |
| 1391 | prompt_logprobs=1, |
| 1392 | ) |
| 1393 | |
| 1394 | |
| 1395 | class BeamSearchParams( |
| 1396 | msgspec.Struct, |
| 1397 | omit_defaults=True, # type: ignore[call-arg] |
| 1398 | # required for @cached_property. |
| 1399 | dict=True, |
| 1400 | ): # type: ignore[call-arg] |
| 1401 | """Beam search parameters for text generation.""" |
| 1402 | |
| 1403 | beam_width: int |
| 1404 | max_tokens: int |
| 1405 | ignore_eos: bool = False |
| 1406 | temperature: float = 0.0 |
| 1407 | length_penalty: float = 1.0 |
| 1408 | include_stop_str_in_output: bool = False |
| 1409 | structured_outputs: StructuredOutputsParams | None = None |
| 1410 | skip_special_tokens: bool = True |
| 1411 | |
| 1412 | def __post_init__(self) -> None: |
| 1413 | _verify_num_sequences(self.beam_width, "beam_width") |