A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Feature] Pydantic validation for speculative.py (#27156)
Signed-off-by: Navya Srivastava <navya.srivastava1707@gmail.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Navya Srivastava committed
Oct 23, 2025 at 05:19 UTC
faee3ccdc21dd6b3a1cfb5ae2146163f6567bdaa
1 file changed
+20
-20
vllm/config/speculative.py
+20
-20
@@ -5,7 +5,7 @@ import ast
5
import hashlib
6
from typing import TYPE_CHECKING, Any, Literal
7
8
-from pydantic import SkipValidation, model_validator
8
+from pydantic import Field, SkipValidation, model_validator
9
from pydantic.dataclasses import dataclass
10
from typing_extensions import Self
11
@@ -62,7 +62,7 @@ class SpeculativeConfig:
62
enforce_eager: bool | None = None
63
"""Override the default enforce_eager from model_config"""
64
# General speculative decoding control
65
- num_speculative_tokens: SkipValidation[int] = None # type: ignore
65
+ num_speculative_tokens: int = Field(default=None, gt=0)
66
"""The number of speculative tokens, if provided. It will default to the
67
number in the draft model config if present, otherwise, it is required."""
68
model: str | None = None
@@ -76,7 +76,7 @@ class SpeculativeConfig:
76
77
If using `ngram` method, the related configuration `prompt_lookup_max` and
78
`prompt_lookup_min` should be considered."""
79
- draft_tensor_parallel_size: int | None = None
79
+ draft_tensor_parallel_size: int | None = Field(default=None, ge=1)
80
"""The degree of the tensor parallelism for the draft model. Can only be 1
81
or the same as the target model's tensor parallel size."""
82
disable_logprobs: bool = True
@@ -89,7 +89,7 @@ class SpeculativeConfig:
89
"""Quantization method that was used to quantize the draft model weights.
90
If `None`, we assume the model weights are not quantized. Note that it only
91
takes effect when using the draft model-based speculative method."""
92
- max_model_len: int | None = None
92
+ max_model_len: int | None = Field(default=None, ge=1)
93
"""The maximum model length of the draft model. Used when testing the
94
ability to skip speculation for some sequences."""
95
revision: str | None = None
@@ -102,7 +102,7 @@ class SpeculativeConfig:
102
will use the default version."""
103
104
# Advanced control
105
- disable_by_batch_size: int | None = None
105
+ disable_by_batch_size: int | None = Field(default=None, ge=2)
106
"""Disable speculative decoding for new incoming requests when the number
107
of enqueued requests is larger than this value, if provided."""
108
disable_padded_drafter_batch: bool = False
@@ -112,10 +112,10 @@ class SpeculativeConfig:
112
only affects the EAGLE method of speculation."""
113
114
# Ngram proposer configuration
115
- prompt_lookup_max: int | None = None
115
+ prompt_lookup_max: int | None = Field(default=None, ge=1)
116
"""Maximum size of ngram token window when using Ngram proposer, required
117
when method is set to ngram."""
118
- prompt_lookup_min: int | None = None
118
+ prompt_lookup_min: int | None = Field(default=None, ge=1)
119
"""Minimum size of ngram token window when using Ngram proposer, if
120
provided. Defaults to 1."""
121
@@ -232,9 +232,8 @@ class SpeculativeConfig:
232
233
if self.model is None and self.num_speculative_tokens is not None:
234
if self.method == "mtp":
235
- assert self.target_model_config is not None, (
236
- "target_model_config must be present for mtp"
237
- )
235
+ if self.target_model_config is None:
236
+ raise ValueError("target_model_config must be present for mtp")
237
if self.target_model_config.hf_text_config.model_type == "deepseek_v32":
238
# FIXME(luccafong): cudgraph with v32 MTP is not supported,
239
# remove this when the issue is fixed.
@@ -268,21 +267,21 @@ class SpeculativeConfig:
267
self.prompt_lookup_min = 5
268
self.prompt_lookup_max = 5
269
elif self.prompt_lookup_min is None:
271
- assert self.prompt_lookup_max is not None
270
+ if self.prompt_lookup_max is None:
271
+ raise ValueError(
272
+ "Either prompt_lookup_max or prompt_lookup_min must be "
273
+ "provided when using the ngram method."
274
+ )
275
self.prompt_lookup_min = self.prompt_lookup_max
276
elif self.prompt_lookup_max is None:
274
- assert self.prompt_lookup_min is not None
277
+ if self.prompt_lookup_min is None:
278
+ raise ValueError(
279
+ "Either prompt_lookup_max or prompt_lookup_min must be "
280
+ "provided when using the ngram method."
281
+ )
282
self.prompt_lookup_max = self.prompt_lookup_min
283
284
# Validate values
278
- if self.prompt_lookup_min < 1:
279
- raise ValueError(
280
- f"prompt_lookup_min={self.prompt_lookup_min} must be > 0"
281
- )
282
- if self.prompt_lookup_max < 1:
283
- raise ValueError(
284
- f"prompt_lookup_max={self.prompt_lookup_max} must be > 0"
285
- )
285
if self.prompt_lookup_min > self.prompt_lookup_max:
286
raise ValueError(
287
f"prompt_lookup_min={self.prompt_lookup_min} must "
@@ -446,6 +445,7 @@ class SpeculativeConfig:
445
self.target_parallel_config, self.draft_tensor_parallel_size
446
)
447
)
448
+ return self
449
450
@staticmethod
451
def _maybe_override_draft_max_model_len(