| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | from collections.abc import Callable |
| 5 | from dataclasses import InitVar |
| 6 | from typing import TYPE_CHECKING, Any, ClassVar, Literal, cast |
| 7 | |
| 8 | from pydantic import Field, field_validator |
| 9 | from typing_extensions import Self |
| 10 | |
| 11 | from vllm.config.utils import config |
| 12 | from vllm.logger import init_logger |
| 13 | from vllm.utils.hashing import safe_hash |
| 14 | from vllm.utils.import_utils import resolve_obj_by_qualname |
| 15 | |
| 16 | if TYPE_CHECKING: |
| 17 | from vllm.v1.core.sched.interface import SchedulerInterface |
| 18 | |
| 19 | logger = init_logger(__name__) |
| 20 | |
| 21 | RunnerType = Literal["generate", "pooling", "draft"] |
| 22 | SchedulerPolicy = Literal["fcfs", "priority"] |
| 23 | |
| 24 | |
| 25 | @config |
| 26 | class SchedulerConfig: |
| 27 | """Scheduler configuration.""" |
| 28 | |
| 29 | max_model_len: InitVar[int] |
| 30 | """Maximum length of a sequence (including prompt and generated text). |
| 31 | |
| 32 | Note: This is stored in the ModelConfig, and is used only here to |
| 33 | provide fallbacks and validate other attributes.""" |
| 34 | |
| 35 | is_encoder_decoder: InitVar[bool] |
| 36 | """True if the model is an encoder-decoder model. |
| 37 | |
| 38 | Note: This is stored in the ModelConfig, and is used only here to |
| 39 | disable chunked prefill and prefix caching for encoder-decoder models. |
| 40 | """ |
| 41 | |
| 42 | DEFAULT_MAX_NUM_BATCHED_TOKENS: ClassVar[int] = 2048 |
| 43 | DEFAULT_MAX_NUM_BATCHED_TOKENS_FOR_BATCHED_DP: ClassVar[int] = 256 |
| 44 | DEFAULT_MAX_NUM_SEQS: ClassVar[int] = 128 |
| 45 | |
| 46 | runner_type: RunnerType = "generate" |
| 47 | """The runner type to launch for the model.""" |
| 48 | |
| 49 | max_num_batched_tokens: int = Field(default=DEFAULT_MAX_NUM_BATCHED_TOKENS, ge=1) |
| 50 | """Maximum number of tokens that can be processed in a single iteration. |
| 51 | |
| 52 | The default value here is mainly for convenience when testing. |
| 53 | In real usage, this should be set in `EngineArgs.create_engine_config`. |
| 54 | """ |
| 55 | |
| 56 | max_num_scheduled_tokens: int | None = Field(default=None, ge=0) |
| 57 | """Maximum number of tokens that the scheduler may issue in a single iteration. |
| 58 | |
| 59 | This is usually equal to max_num_batched_tokens, but can be smaller in cases |
| 60 | when the model might append tokens into the batch (such as speculative decoding). |
| 61 | Defaults to max_num_batched_tokens.""" |
| 62 | |
| 63 | max_num_seqs: int = Field(default=DEFAULT_MAX_NUM_SEQS, ge=1) |
| 64 | """Maximum number of sequences to be processed in a single iteration. |
| 65 | |
| 66 | The default value here is mainly for convenience when testing. |
| 67 | In real usage, this should be set in `EngineArgs.create_engine_config`. |
| 68 | """ |
| 69 | |
| 70 | max_num_active_seqs: int | None = Field(default=None, ge=1) |
| 71 | """Maximum number of requests the scheduler admits into RUNNING. |
| 72 | |
| 73 | ``max_num_seqs`` sizes the model runner (per-request buffers and CUDA |
| 74 | graph capture) and is also the default admission limit. Setting this |
| 75 | lowers only the number of requests that may occupy RUNNING, so decode |
| 76 | batches stay smaller without shrinking runner or graph capacity. Must |
| 77 | be ``<= max_num_seqs``. ``None`` (default) keeps current behavior. |
| 78 | """ |
| 79 | |
| 80 | long_prefill_token_threshold: int = Field(default=0, ge=0) |
| 81 | """For chunked prefill, a request is considered long if the prompt is |
| 82 | longer than this number of tokens. 0 disables the cap (default). |
| 83 | |
| 84 | The cap is not applied when the request is the only one in the batch, |
| 85 | since there is no other request for it to starve.""" |
| 86 | |
| 87 | long_prefill_token_threshold_adaptive: bool = Field(default=False) |
| 88 | """Floor the effective long prefill token threshold at a fair share of |
| 89 | the token budget: max_num_batched_tokens divided by the number of |
| 90 | queued and running requests. Only applies when |
| 91 | long_prefill_token_threshold is nonzero.""" |
| 92 | |
| 93 | max_num_queued_reqs: int | None = Field(default=None, ge=0) |
| 94 | """Maximum number of requests that can be in-flight (waiting or running) |
| 95 | at the same time, or None for no limit. When the limit is reached, new |
| 96 | requests are rejected with HTTP 503 so the client can retry on another |
| 97 | instance. This bounds vLLM's otherwise unbounded request queue and is |
| 98 | primarily a coarse capacity valve. |
| 99 | |
| 100 | Unlike ``max_num_seqs``, which applies per data-parallel rank, this |
| 101 | limit is enforced in the API server process and counts in-flight |
| 102 | requests across all DP ranks it routes to. Size it as roughly |
| 103 | ``data_parallel_size * max_num_seqs`` plus the desired queue depth if |
| 104 | it should not bind before per-rank admission does.""" |
| 105 | |
| 106 | max_num_queued_tokens: int | None = Field(default=None, ge=0) |
| 107 | """Maximum total prompt tokens of requests currently in the prefill |
| 108 | phase, or None for no limit. When the limit is reached, new requests |
| 109 | are rejected with HTTP 503. |
| 110 | |
| 111 | This is a TTFT QoS mechanism: by setting it to |
| 112 | ``target_TTFT * prefill_throughput`` you reject requests when the |
| 113 | prefill backlog would exceed the latency target. In a disaggregated |
| 114 | prefill-decode setup this maps directly to the prefill pool's |
| 115 | capacity. |
| 116 | |
| 117 | Like ``max_num_queued_reqs``, this limit is enforced in the API |
| 118 | server process and covers the prefill backlog across all DP ranks it |
| 119 | routes to, so ``prefill_throughput`` in the formula above is the |
| 120 | aggregate throughput of the deployment. |
| 121 | |
| 122 | Note: the count is conservative. A partially prefilled request |
| 123 | still contributes its full ``prompt_len`` until it transitions out |
| 124 | of the prefill phase, because the scheduler's per-iteration |
| 125 | ``num_computed_tokens`` progress is not propagated to the API |
| 126 | server process during prefill (``EngineCoreOutput`` is only |
| 127 | emitted once the request starts producing tokens). Similarly, |
| 128 | prefix-cache hits (``num_cached_tokens``) are only known to the |
| 129 | OutputProcessor after prefill completes. This overestimates the |
| 130 | real backlog, causing earlier rejection than strictly necessary |
| 131 | — the safe direction for QoS. The impact is limited to long |
| 132 | prompts under chunked prefill; short prompts that prefill in a |
| 133 | single iteration are unaffected.""" |
| 134 | |
| 135 | enable_chunked_prefill: bool = True |
| 136 | """If True, prefill requests can be chunked based |
| 137 | on the remaining `max_num_batched_tokens`. |
| 138 | |
| 139 | The default value here is mainly for convenience when testing. |
| 140 | In real usage, this should be set in `EngineArgs.create_engine_config`. |
| 141 | """ |
| 142 | |
| 143 | is_multimodal_model: bool = False |
| 144 | """True if the model is multimodal.""" |
| 145 | |
| 146 | # TODO (ywang96): Make this configurable. |
| 147 | max_num_encoder_input_tokens: int = Field(init=False) |
| 148 | """Multimodal encoder compute budget, only used in V1. |
| 149 | |
| 150 | NOTE: This is not currently configurable. It will be overridden by |
| 151 | max_num_batched_tokens in case max multimodal embedding size is larger.""" |
| 152 | |
| 153 | # TODO (ywang96): Make this configurable. |
| 154 | encoder_cache_size: int = Field(init=False) |
| 155 | """Multimodal encoder cache size, only used in V1. |
| 156 | |
| 157 | NOTE: This is not currently configurable. It will be overridden by |
| 158 | max_num_batched_tokens in case max multimodal embedding size is larger.""" |
| 159 | |
| 160 | policy: SchedulerPolicy = "fcfs" |
| 161 | """The scheduling policy to use: |
| 162 | |
| 163 | - "fcfs" means first come first served, i.e. requests are handled in order |
| 164 | of arrival. |
| 165 | - "priority" means requests are handled based on given priority (lower |
| 166 | value means earlier handling) and time of arrival deciding any ties).""" |
| 167 | |
| 168 | disable_chunked_mm_input: bool = False |
| 169 | """If set to true and chunked prefill is enabled, we do not want to |
| 170 | partially schedule a multimodal item. Only used in V1 |
| 171 | This ensures that if a request has a mixed prompt |
| 172 | (like text tokens TTTT followed by image tokens IIIIIIIIII) where only |
| 173 | some image tokens can be scheduled (like TTTTIIIII, leaving IIIII), |
| 174 | it will be scheduled as TTTT in one step and IIIIIIIIII in the next.""" |
| 175 | |
| 176 | # scheduler class or path. "vllm.v1.core.sched.scheduler.Scheduler" |
| 177 | # (default) or "mod.custom_class". |
| 178 | scheduler_cls: str | type[object] | None = None |
| 179 | """The scheduler class to use. "vllm.v1.core.sched.scheduler.Scheduler" is |
| 180 | the default scheduler. Can be a class directly or the path to a class of |
| 181 | form "mod.custom_class".""" |
| 182 | |
| 183 | disable_hybrid_kv_cache_manager: bool | None = None |
| 184 | """If set to True, KV cache manager will allocate the same size of KV cache |
| 185 | for all attention layers even if there are multiple type of attention layers |
| 186 | like full attention and sliding window attention. |
| 187 | If set to None, the default value will be determined based on the environment |
| 188 | and starting configuration. |
| 189 | """ |
| 190 | |
| 191 | scheduler_reserve_full_isl: bool = True |
| 192 | """If True, the scheduler checks whether the full input sequence length |
| 193 | fits in the KV cache before admitting a new request, rather than only |
| 194 | checking the first chunk. Prevents over-admission and KV cache thrashing |
| 195 | with chunked prefill.""" |
| 196 | |
| 197 | watermark: float = Field(default=0.0, ge=0.0, lt=1.0) |
| 198 | """Fraction of total KV cache blocks to keep free (the watermark) when |
| 199 | admitting waiting or preempted requests into the running queue. This headroom |
| 200 | helps avoid frequent KV cache eviction and the resulting repeated preemption |
| 201 | of requests when GPU memory is scarce. Must be in the range [0.0, 1.0); 0.0 |
| 202 | (the default) disables the watermark.""" |
| 203 | |
| 204 | prefill_schedule_interval: int = Field(default=1, ge=1) |
| 205 | """For data-parallel deployments, only admit new prefill requests |
| 206 | once every N engine steps, aligned across DP ranks, to better balance |
| 207 | per-step forward-pass times.""" |
| 208 | |
| 209 | async_scheduling: bool | None = None |
| 210 | """If set to False, disable async scheduling. Async scheduling helps to |
| 211 | avoid gaps in GPU utilization, leading to better latency and throughput. |
| 212 | """ |
| 213 | |
| 214 | stream_interval: int = Field(default=1, ge=1) |
| 215 | """The interval (or buffer size) for streaming in terms of token length. |
| 216 | A smaller value (1) makes streaming smoother by sending each token immediately, |
| 217 | while a larger value (e.g., 10) reduces host overhead and may increase throughput |
| 218 | by batching multiple tokens before sending.""" |
| 219 | |
| 220 | @staticmethod |
| 221 | def default_factory(**kwargs): |
| 222 | """Create a `SchedulerConfig` with default values for its `InitVar`s.""" |
| 223 | if "max_model_len" not in kwargs: |
| 224 | kwargs["max_model_len"] = 8192 |
| 225 | if "is_encoder_decoder" not in kwargs: |
| 226 | kwargs["is_encoder_decoder"] = False |
| 227 | return SchedulerConfig(**kwargs) |
| 228 | |
| 229 | def get_scheduler_cls(self) -> type["SchedulerInterface"]: |
| 230 | if self.scheduler_cls is None: |
| 231 | if self.async_scheduling: |
| 232 | from vllm.v1.core.sched.async_scheduler import AsyncScheduler |
| 233 | |
| 234 | return AsyncScheduler |
| 235 | from vllm.v1.core.sched.scheduler import Scheduler |
| 236 | |
| 237 | return Scheduler |
| 238 | |
| 239 | # The first half of this warning can be removed once the Scheduler interface is |
| 240 | # finalized and we can maintain support for scheduler classes that implement it |
| 241 | if not ( |
| 242 | isinstance(self.scheduler_cls, str) |
| 243 | and self.scheduler_cls.startswith("vllm.") |
| 244 | ): |
| 245 | logger.warning_once( |
| 246 | "Using custom scheduler class %s. This scheduler interface is not " |
| 247 | "public and compatibility may not be maintained. If you have " |
| 248 | "subclassed Scheduler instead of AsyncScheduler, you will see " |
| 249 | "degraded performance due to async scheduling being disabled.", |
| 250 | self.scheduler_cls, # type: ignore[arg-type] |
| 251 | ) |
| 252 | if not isinstance(self.scheduler_cls, str): |
| 253 | return cast(type["SchedulerInterface"], self.scheduler_cls) |
| 254 | return resolve_obj_by_qualname(self.scheduler_cls) |
| 255 | |
| 256 | def compute_hash(self) -> str: |
| 257 | """WARNING: Whenever a new field is added to this config, |
| 258 | ensure that it is included in the factors list if |
| 259 | it affects the computation graph. |
| 260 | |
| 261 | Provide a hash that uniquely identifies all the configs |
| 262 | that affect the structure of the computation |
| 263 | graph from input ids/embeddings to the final hidden states, |
| 264 | excluding anything before input ids/embeddings and after |
| 265 | the final hidden states. |
| 266 | """ |
| 267 | factors: list[Any] = [] |
| 268 | |
| 269 | # max_num_batched_tokens need to be included in the hash due |
| 270 | # to two reasons: |
| 271 | # 1. LoRA creates static buffers based on max_num_batched_tokens. |
| 272 | # The tensor sizes and strides get captured in the torch.compile |
| 273 | # graph explicitly. |
| 274 | # 2. Inductor decides whether using 32-bit or 64-bit indexing integer |
| 275 | # based on the data sizes. `max_num_batched_tokens` has an |
| 276 | # impact on that. For more details, please check |
| 277 | # https://github.com/vllm-project/vllm/issues/29585 |
| 278 | factors.append(self.max_num_batched_tokens) |
| 279 | |
| 280 | # PLE and other model components allocate static per-request buffers. |
| 281 | # Their shapes are captured in compiled graphs. |
| 282 | factors.append(self.max_num_seqs) |
| 283 | |
| 284 | hash_str = safe_hash(str(factors).encode(), usedforsecurity=False).hexdigest() |
| 285 | return hash_str |
| 286 | |
| 287 | @field_validator("scheduler_cls", "async_scheduling", mode="wrap") |
| 288 | @classmethod |
| 289 | def _skip_none_validation(cls, value: Any, handler: Callable) -> Any: |
| 290 | """Skip validation if the value is `None` when initialisation is delayed.""" |
| 291 | return None if value is None else handler(value) |
| 292 | |
| 293 | def __post_init__(self, max_model_len: int, is_encoder_decoder: bool) -> None: |
| 294 | if is_encoder_decoder: |
| 295 | # Chunked prefill should be disabled for encoder-decoder models. |
| 296 | self.disable_chunked_mm_input = True |
| 297 | self.enable_chunked_prefill = False |
| 298 | self.long_prefill_token_threshold = 0 |
| 299 | logger.info( |
| 300 | "Encoder-decoder models do not support chunked prefill nor" |
| 301 | " prefix caching; disabling both." |
| 302 | ) |
| 303 | |
| 304 | self.max_num_encoder_input_tokens = self.max_num_batched_tokens |
| 305 | self.encoder_cache_size = self.max_num_batched_tokens |
| 306 | |
| 307 | if self.enable_chunked_prefill: |
| 308 | logger.info_once( |
| 309 | "Chunked prefill is enabled with max_num_batched_tokens=%d.", |
| 310 | self.max_num_batched_tokens, |
| 311 | ) |
| 312 | |
| 313 | self.verify_max_model_len(max_model_len) |
| 314 | |
| 315 | def verify_max_model_len(self, max_model_len: int) -> Self: |
| 316 | if ( |
| 317 | self.max_num_batched_tokens < max_model_len |
| 318 | and not self.enable_chunked_prefill |
| 319 | ): |
| 320 | raise ValueError( |
| 321 | f"max_num_batched_tokens ({self.max_num_batched_tokens}) is " |
| 322 | f"smaller than max_model_len ({max_model_len}). " |
| 323 | "This effectively limits the maximum sequence length to " |
| 324 | "max_num_batched_tokens and makes vLLM reject longer " |
| 325 | "sequences. Please increase max_num_batched_tokens or " |
| 326 | "decrease max_model_len." |
| 327 | ) |
| 328 | |
| 329 | if self.max_num_batched_tokens < self.max_num_seqs: |
| 330 | raise ValueError( |
| 331 | f"max_num_batched_tokens ({self.max_num_batched_tokens}) must " |
| 332 | "be greater than or equal to max_num_seqs " |
| 333 | f"({self.max_num_seqs})." |
| 334 | ) |
| 335 | |
| 336 | if ( |
| 337 | self.max_num_active_seqs is not None |
| 338 | and self.max_num_active_seqs > self.max_num_seqs |
| 339 | ): |
| 340 | raise ValueError( |
| 341 | f"max_num_active_seqs ({self.max_num_active_seqs}) cannot be " |
| 342 | f"greater than max_num_seqs ({self.max_num_seqs})." |
| 343 | ) |
| 344 | |
| 345 | if self.max_num_batched_tokens > self.max_num_seqs * max_model_len: |
| 346 | logger.warning( |
| 347 | "max_num_batched_tokens (%d) exceeds max_num_seqs " |
| 348 | "* max_model_len (%d). This may lead to unexpected behavior.", |
| 349 | self.max_num_batched_tokens, |
| 350 | self.max_num_seqs * max_model_len, |
| 351 | ) |
| 352 | |
| 353 | if self.long_prefill_token_threshold > max_model_len: |
| 354 | raise ValueError( |
| 355 | "long_prefill_token_threshold " |
| 356 | f"({self.long_prefill_token_threshold}) cannot be greater " |
| 357 | f"than the max_model_len ({max_model_len})." |
| 358 | ) |
| 359 | |
| 360 | return self |