A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 10:00 UTC
[Refactor] Move validation to params definitions (#34362)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Feb 12, 2026 at 11:33 UTC
ced2a92f40ed56148a6f4496239b55a65f854081
3 files changed
+264
-245
vllm/pooling_params.py
+7
-10
@@ -72,7 +72,7 @@ class PoolingParams(
72
"""Returns a deep copy of the PoolingParams instance."""
73
return deepcopy(self)
74
75
- def verify(self, model_config: "ModelConfig") -> None:
75
+ def verify(self, model_config: ModelConfig) -> None:
76
# plugin task uses io_processor.parse_request to verify inputs,
77
# skipping PoolingParams verify
78
if self.task == "plugin":
@@ -87,12 +87,7 @@ class PoolingParams(
87
self._set_default_parameters(model_config)
88
self._verify_valid_parameters()
89
90
- def _merge_default_parameters(
91
- self, model_config: "ModelConfig | None" = None
92
- ) -> None:
93
- if model_config is None:
94
- return
95
-
90
+ def _merge_default_parameters(self, model_config: ModelConfig) -> None:
91
pooler_config = model_config.pooler_config
92
if pooler_config is None:
93
return
@@ -119,7 +114,9 @@ class PoolingParams(
114
self._verify_step_pooling(pooler_config, valid_parameters)
115
116
def _verify_step_pooling(
122
- self, pooler_config: "PoolerConfig", valid_parameters: list[str]
117
+ self,
118
+ pooler_config: PoolerConfig,
119
+ valid_parameters: list[str],
120
):
121
step_pooling_parameters = ["step_tag_id", "returned_token_ids"]
122
if pooler_config.tok_pooling_type != "STEP":
@@ -142,12 +139,12 @@ class PoolingParams(
139
if getattr(self, k, None) is None:
140
setattr(self, k, getattr(pooler_config, k))
141
145
- def _set_default_parameters(self, model_config: "ModelConfig | None"):
142
+ def _set_default_parameters(self, model_config: ModelConfig):
143
if self.task in ["embed", "token_embed"]:
144
if self.use_activation is None:
145
self.use_activation = True
146
150
- if self.dimensions is not None and model_config is not None:
147
+ if self.dimensions is not None:
148
if not model_config.is_matryoshka:
149
raise ValueError(
150
f'Model "{model_config.served_model_name}" does not '
vllm/sampling_params.py
+238
@@ -3,6 +3,7 @@
3
"""Sampling parameters for text generation."""
4
5
import copy
6
+import json
7
from dataclasses import field
8
from enum import Enum, IntEnum
9
from functools import cached_property
@@ -11,6 +12,7 @@ from typing import Annotated, Any
12
import msgspec
13
from pydantic.dataclasses import dataclass
14
15
+from vllm.config import ModelConfig, SpeculativeConfig, StructuredOutputsConfig
16
from vllm.exceptions import VLLMValidationError
17
from vllm.logger import init_logger
18
from vllm.logits_process import LogitsProcessor
@@ -453,6 +455,11 @@ class SamplingParams(
455
parameter="prompt_logprobs",
456
value=self.prompt_logprobs,
457
)
458
+ if self.logits_processors:
459
+ # TODO: Remove `logits_processors` attribute
460
+ raise ValueError(
461
+ "vLLM V1 does not support per request user-provided logits processors."
462
+ )
463
if self.truncate_prompt_tokens is not None and (
464
self.truncate_prompt_tokens == 0 or self.truncate_prompt_tokens < -1
465
):
@@ -589,6 +596,237 @@ class SamplingParams(
596
)
597
return copy.deepcopy(self, memo=logit_processor_refs)
598
599
+ def verify(
600
+ self,
601
+ model_config: ModelConfig,
602
+ speculative_config: SpeculativeConfig | None,
603
+ structured_outputs_config: StructuredOutputsConfig | None,
604
+ tokenizer: TokenizerLike | None,
605
+ ) -> None:
606
+ self._validate_logprobs(model_config)
607
+ self._validate_logit_bias(model_config)
608
+ self._validate_allowed_token_ids(tokenizer)
609
+ self._validate_spec_decode(speculative_config)
610
+ self._validate_structured_outputs(structured_outputs_config, tokenizer)
611
+
612
+ def _validate_logprobs(self, model_config: ModelConfig) -> None:
613
+ max_logprobs = model_config.max_logprobs
614
+ if max_logprobs == -1:
615
+ max_logprobs = model_config.get_vocab_size()
616
+
617
+ # Validate sample logprobs.
618
+ if num_logprobs := self.logprobs:
619
+ if num_logprobs == -1:
620
+ num_logprobs = model_config.get_vocab_size()
621
+ if num_logprobs > max_logprobs:
622
+ raise VLLMValidationError(
623
+ f"Requested sample logprobs of {num_logprobs}, "
624
+ f"which is greater than max allowed: {max_logprobs}",
625
+ parameter="logprobs",
626
+ value=num_logprobs,
627
+ )
628
+
629
+ # Validate prompt logprobs.
630
+ if num_prompt_logprobs := self.prompt_logprobs:
631
+ if num_prompt_logprobs == -1:
632
+ num_prompt_logprobs = model_config.get_vocab_size()
633
+ if num_prompt_logprobs > max_logprobs:
634
+ raise VLLMValidationError(
635
+ f"Requested prompt logprobs of {num_prompt_logprobs}, "
636
+ f"which is greater than max allowed: {max_logprobs}",
637
+ parameter="prompt_logprobs",
638
+ value=num_prompt_logprobs,
639
+ )
640
+
641
+ def _validate_logit_bias(self, model_config: ModelConfig) -> None:
642
+ """Validate logit_bias token IDs are within vocabulary range."""
643
+ if not self.logit_bias:
644
+ return
645
+
646
+ vocab_size = model_config.get_vocab_size()
647
+ invalid_token_ids = [
648
+ token_id
649
+ for token_id in self.logit_bias
650
+ if token_id < 0 or token_id >= vocab_size
651
+ ]
652
+
653
+ if invalid_token_ids:
654
+ raise VLLMValidationError(
655
+ f"token_id(s) {invalid_token_ids} in logit_bias contain "
656
+ f"out-of-vocab token ids. Vocabulary size: {vocab_size}",
657
+ parameter="logit_bias",
658
+ value=invalid_token_ids,
659
+ )
660
+
661
+ def _validate_allowed_token_ids(self, tokenizer: TokenizerLike | None) -> None:
662
+ allowed_token_ids = self.allowed_token_ids
663
+ if allowed_token_ids is None:
664
+ return
665
+
666
+ if len(allowed_token_ids) == 0:
667
+ raise VLLMValidationError(
668
+ "allowed_token_ids is not None and empty!",
669
+ parameter="allowed_token_ids",
670
+ value=allowed_token_ids,
671
+ )
672
+
673
+ if tokenizer is not None:
674
+ vocab_size = len(tokenizer)
675
+ invalid_token_ids = [
676
+ token_id
677
+ for token_id in allowed_token_ids
678
+ if token_id < 0 or token_id >= vocab_size
679
+ ]
680
+ if invalid_token_ids:
681
+ raise VLLMValidationError(
682
+ "allowed_token_ids contains out-of-vocab token id!",
683
+ parameter="allowed_token_ids",
684
+ value=invalid_token_ids,
685
+ )
686
+
687
+ def _validate_spec_decode(
688
+ self,
689
+ speculative_config: SpeculativeConfig | None,
690
+ ) -> None:
691
+ if speculative_config is None:
692
+ return
693
+
694
+ # Some sampling parameters are not yet compatible with spec decoding.
695
+ if self.min_tokens > 1 or self.min_p > _SAMPLING_EPS or self.logit_bias:
696
+ raise ValueError(
697
+ "The min_tokens, min_p, and logit_bias sampling parameters "
698
+ "are not yet supported with speculative decoding."
699
+ )
700
+
701
+ def _validate_structured_outputs(
702
+ self,
703
+ structured_outputs_config: StructuredOutputsConfig | None,
704
+ tokenizer: TokenizerLike | None,
705
+ ) -> None:
706
+ if structured_outputs_config is None or self.structured_outputs is None:
707
+ return
708
+
709
+ if tokenizer is None:
710
+ raise ValueError(
711
+ "Structured outputs requires a tokenizer so it can't be used with 'skip_tokenizer_init'" # noqa: E501
712
+ )
713
+
714
+ backend = structured_outputs_config.backend
715
+ if _backend := self.structured_outputs._backend:
716
+ # Request-level backend selection is not supported.
717
+ # The values may differ if `params` is reused and was set
718
+ # to a specific backend based on `auto` behavior in a previous
719
+ # request. We remember that it was set as a result of `auto`
720
+ # using the `_backend_was_auto` field set in the params.
721
+ if backend != _backend and not (
722
+ backend == "auto" and self.structured_outputs._backend_was_auto
723
+ ):
724
+ raise ValueError(
725
+ "Request-level structured output backend selection is not "
726
+ f"supported. The request specified '{_backend}', but vLLM "
727
+ f"was initialised with '{backend}'. This error can be "
728
+ "resolved by removing '_backend' from the request."
729
+ )
730
+ else:
731
+ self.structured_outputs._backend = backend
732
+
733
+ # Request content validation
734
+ if (
735
+ isinstance(self.structured_outputs.choice, list)
736
+ and not self.structured_outputs.choice
737
+ ):
738
+ # It is invalid for choice to be an empty list
739
+ raise ValueError(
740
+ f"Choice '{self.structured_outputs.choice}' cannot be an empty list" # noqa: E501
741
+ )
742
+ # Reject empty string grammar early to avoid engine-side crashes
743
+ if (
744
+ isinstance(self.structured_outputs.grammar, str)
745
+ and self.structured_outputs.grammar.strip() == ""
746
+ ):
747
+ raise ValueError("structured_outputs.grammar cannot be an empty string")
748
+
749
+ from vllm.tokenizers.mistral import MistralTokenizer
750
+ from vllm.v1.structured_output.backend_guidance import (
751
+ has_guidance_unsupported_json_features,
752
+ validate_guidance_grammar,
753
+ )
754
+ from vllm.v1.structured_output.backend_lm_format_enforcer import (
755
+ validate_structured_output_request_lm_format_enforcer,
756
+ )
757
+ from vllm.v1.structured_output.backend_outlines import (
758
+ validate_structured_output_request_outlines,
759
+ )
760
+ from vllm.v1.structured_output.backend_xgrammar import validate_xgrammar_grammar
761
+
762
+ if backend.startswith("xgrammar"):
763
+ # xgrammar with no fallback
764
+ validate_xgrammar_grammar(self)
765
+ elif backend.startswith("guidance"):
766
+ # TODO: ideally we would have the LLTokenizer here as Lark syntax
767
+ # allows <|special_token|> and similar, see
768
+ # https://github.com/guidance-ai/llguidance/blob/main/docs/syntax.md#special-tokens
769
+ # Without tokenizer these are disallowed in grammars.
770
+ if isinstance(tokenizer, MistralTokenizer):
771
+ raise ValueError(
772
+ "Mistral tokenizer is not supported for the 'guidance' "
773
+ "structured output backend. Please use ['xgrammar', 'outlines'] "
774
+ "backends or tokenizer_mode='hf' instead."
775
+ )
776
+ validate_guidance_grammar(self, tokenizer=None)
777
+ elif backend == "outlines":
778
+ # outlines backend
779
+ validate_structured_output_request_outlines(self)
780
+ elif backend == "lm-format-enforcer":
781
+ # lm format enforcer backend
782
+ if isinstance(tokenizer, MistralTokenizer):
783
+ raise ValueError(
784
+ "Mistral tokenizer is not supported for the 'lm-format-enforcer' "
785
+ "structured output backend. Please use ['xgrammar', 'outlines'] "
786
+ "backends or tokenizer_mode='hf' instead."
787
+ )
788
+ validate_structured_output_request_lm_format_enforcer(self)
789
+ else:
790
+ # NOTE: backend must be "auto" here, because we have
791
+ # checked supported_backends above.
792
+ # In this mode, we set opinionated defaults based on what we think
793
+ # will satisfy the most use cases without having to worry about
794
+ # this setting. We include fallback behavior here, but not with any
795
+ # other setting where a specific backend was specified.
796
+ try:
797
+ validate_xgrammar_grammar(self)
798
+ self.structured_outputs._backend = "xgrammar"
799
+ except ValueError:
800
+ # The request either failed validation
801
+ # or includes some jsonschema feature(s) that
802
+ # are not supported in xgrammar.
803
+
804
+ # Check if schema has features unsupported by guidance
805
+ so_params = self.structured_outputs
806
+ skip_guidance = False
807
+ if so_params.json:
808
+ if isinstance(so_params.json, str):
809
+ schema = json.loads(so_params.json)
810
+ else:
811
+ schema = so_params.json
812
+ skip_guidance = has_guidance_unsupported_json_features(schema)
813
+
814
+ if isinstance(tokenizer, MistralTokenizer) or skip_guidance:
815
+ # Fall back to outlines if the tokenizer is Mistral
816
+ # or if schema contains features unsupported by guidance
817
+ validate_structured_output_request_outlines(self)
818
+ self.structured_outputs._backend = "outlines"
819
+ else:
820
+ # Fall back to guidance by default.
821
+ validate_guidance_grammar(self, tokenizer=None)
822
+ self.structured_outputs._backend = "guidance"
823
+ # Remember that this backend was set automatically
824
+ self.structured_outputs._backend_was_auto = True
825
+
826
+ # Run post-init validation. This is also important to ensure subsequent
827
+ # roundtrip serialization/deserialization won't fail.
828
+ self.structured_outputs.__post_init__()
829
+
830
def __repr__(self) -> str:
831
return (
832
f"SamplingParams(n={self.n}, "
vllm/v1/engine/input_processor.py
+19
-235
@@ -6,7 +6,6 @@ from collections.abc import Mapping
6
from typing import Any, Literal, cast
7
8
from vllm.config import VllmConfig
9
-from vllm.exceptions import VLLMValidationError
9
from vllm.inputs.data import (
10
ProcessorInputs,
11
PromptType,
@@ -30,25 +29,13 @@ from vllm.multimodal.utils import argsort_mm_positions
29
from vllm.pooling_params import PoolingParams
30
from vllm.renderers import BaseRenderer
31
from vllm.renderers.inputs import DictPrompt, TokPrompt
33
-from vllm.sampling_params import _SAMPLING_EPS, SamplingParams
32
+from vllm.sampling_params import SamplingParams
33
from vllm.tasks import POOLING_TASKS, SupportedTask
34
from vllm.tokenizers import TokenizerLike
36
-from vllm.tokenizers.mistral import MistralTokenizer
35
from vllm.utils import length_from_prompt_token_ids_or_embeds, random_uuid
36
from vllm.utils.torch_utils import set_default_torch_num_threads
37
from vllm.v1.engine import EngineCoreRequest
38
from vllm.v1.metrics.stats import MultiModalCacheStats
41
-from vllm.v1.structured_output.backend_guidance import (
42
- has_guidance_unsupported_json_features,
43
- validate_guidance_grammar,
44
-)
45
-from vllm.v1.structured_output.backend_lm_format_enforcer import (
46
- validate_structured_output_request_lm_format_enforcer,
47
-)
48
-from vllm.v1.structured_output.backend_outlines import (
49
- validate_structured_output_request_outlines,
50
-)
51
-from vllm.v1.structured_output.backend_xgrammar import validate_xgrammar_grammar
39
40
logger = init_logger(__name__)
41
@@ -64,6 +51,7 @@ class InputProcessor:
51
self.cache_config = vllm_config.cache_config
52
self.lora_config = vllm_config.lora_config
53
self.scheduler_config = vllm_config.scheduler_config
54
+ self.speculative_config = vllm_config.speculative_config
55
self.structured_outputs_config = vllm_config.structured_outputs_config
56
self.observability_config = vllm_config.observability_config
57
@@ -101,101 +89,6 @@ class InputProcessor:
89
def renderer(self) -> BaseRenderer:
90
return self.input_preprocessor.renderer
91
104
- def _validate_logprobs(
105
- self,
106
- params: SamplingParams,
107
- ) -> None:
108
- max_logprobs = self.model_config.max_logprobs
109
- if max_logprobs == -1:
110
- max_logprobs = self.model_config.get_vocab_size()
111
-
112
- # Validate sample logprobs.
113
- if params.logprobs:
114
- num_logprobs = params.logprobs
115
- if num_logprobs == -1:
116
- num_logprobs = self.model_config.get_vocab_size()
117
- if num_logprobs > max_logprobs:
118
- raise VLLMValidationError(
119
- f"Requested sample logprobs of {num_logprobs}, "
120
- f"which is greater than max allowed: {max_logprobs}",
121
- parameter="logprobs",
122
- value=num_logprobs,
123
- )
124
-
125
- # Validate prompt logprobs.
126
- if params.prompt_logprobs:
127
- num_prompt_logprobs = params.prompt_logprobs
128
- if num_prompt_logprobs == -1:
129
- num_prompt_logprobs = self.model_config.get_vocab_size()
130
- if num_prompt_logprobs > max_logprobs:
131
- raise VLLMValidationError(
132
- f"Requested prompt logprobs of {num_prompt_logprobs}, "
133
- f"which is greater than max allowed: {max_logprobs}",
134
- parameter="prompt_logprobs",
135
- value=num_prompt_logprobs,
136
- )
137
-
138
- def _validate_sampling_params(
139
- self,
140
- params: SamplingParams,
141
- ) -> None:
142
- self._validate_structured_output(params)
143
- self._validate_logit_bias(params)
144
-
145
- if params.allowed_token_ids is None:
146
- return
147
- if not params.allowed_token_ids:
148
- raise ValueError("allowed_token_ids is not None and empty!")
149
- if self.tokenizer is None:
150
- # When skip_tokenizer_init=True, we can't validate token IDs
151
- # Skip validation and let the model handle invalid tokens
152
- return
153
- vocab_size = len(self.tokenizer)
154
- if not all(0 <= tid < vocab_size for tid in params.allowed_token_ids):
155
- raise ValueError("allowed_token_ids contains out-of-vocab token id!")
156
-
157
- def _validate_logit_bias(
158
- self,
159
- params: SamplingParams,
160
- ) -> None:
161
- """Validate logit_bias token IDs are within vocabulary range."""
162
- if not params.logit_bias:
163
- return
164
-
165
- vocab_size = self.model_config.get_vocab_size()
166
- invalid_token_ids = []
167
-
168
- for token_id in params.logit_bias:
169
- if token_id < 0 or token_id >= vocab_size:
170
- invalid_token_ids.append(token_id)
171
-
172
- if invalid_token_ids:
173
- raise VLLMValidationError(
174
- f"token_id(s) {invalid_token_ids} in logit_bias contain "
175
- f"out-of-vocab token ids. Vocabulary size: {vocab_size}",
176
- parameter="logit_bias",
177
- value=invalid_token_ids,
178
- )
179
-
180
- def _validate_supported_sampling_params(
181
- self,
182
- params: SamplingParams,
183
- ) -> None:
184
- # Logits processors not supported.
185
- if params.logits_processors:
186
- raise ValueError(
187
- "vLLM V1 does not support per request user-provided logits processors."
188
- )
189
-
190
- # Some sampling parameters are not yet compatible with spec decoding.
191
- if self.vllm_config.speculative_config is not None and (
192
- params.min_tokens > 1 or params.min_p > _SAMPLING_EPS or params.logit_bias
193
- ):
194
- raise ValueError(
195
- "The min_tokens, min_p, and logit_bias sampling parameters "
196
- "are not yet supported with speculative decoding."
197
- )
198
-
92
def _validate_params(
93
self,
94
params: SamplingParams | PoolingParams,
@@ -203,11 +96,15 @@ class InputProcessor:
96
# is passed to all `process_inputs` calls
97
supported_tasks: tuple[SupportedTask, ...] | None,
98
):
206
- """
207
- Validate supported SamplingParam.
208
- Should raise ValueError if unsupported for API Server.
209
- """
210
- if isinstance(params, PoolingParams):
99
+ """Raise `ValueError` if SamplingParams or PoolingParams is not valid."""
100
+ if isinstance(params, SamplingParams):
101
+ params.verify(
102
+ self.model_config,
103
+ self.speculative_config,
104
+ self.structured_outputs_config,
105
+ self.tokenizer,
106
+ )
107
+ elif isinstance(params, PoolingParams):
108
if supported_tasks is None:
109
raise RuntimeError("`supported_tasks` must be passed for pooling")
110
@@ -233,12 +130,11 @@ class InputProcessor:
130
)
131
132
params.verify(self.model_config)
236
-
237
- return
238
-
239
- self._validate_logprobs(params)
240
- self._validate_sampling_params(params)
241
- self._validate_supported_sampling_params(params)
133
+ else:
134
+ raise TypeError(
135
+ f"params must be either SamplingParams or PoolingParams, "
136
+ f"but got {type(params).__name__}"
137
+ )
138
139
def _parse_mm_items(self, mm_data: MultiModalDataDict) -> MultiModalDataItems:
140
mm_processor = self.input_preprocessor._get_mm_processor()
@@ -334,120 +230,6 @@ class InputProcessor:
230
"[lora_path]` to use the LoRA tokenizer."
231
)
232
337
- def _validate_structured_output(self, params: SamplingParams) -> None:
338
- if not params.structured_outputs or not self.structured_outputs_config:
339
- return
340
-
341
- if self.model_config.skip_tokenizer_init and params.structured_outputs:
342
- raise ValueError(
343
- "Structured outputs requires a tokenizer so it can't be used with 'skip_tokenizer_init'" # noqa: E501
344
- )
345
-
346
- backend = self.structured_outputs_config.backend
347
- if _backend := params.structured_outputs._backend:
348
- # Request-level backend selection is not supported.
349
- # The values may differ if `params` is reused and was set
350
- # to a specific backend based on `auto` behavior in a previous
351
- # request. We remember that it was set as a result of `auto`
352
- # using the `_backend_was_auto` field set in the params.
353
- if backend != _backend and not (
354
- backend == "auto" and params.structured_outputs._backend_was_auto
355
- ):
356
- raise ValueError(
357
- "Request-level structured output backend selection is not "
358
- f"supported. The request specified '{_backend}', but vLLM "
359
- f"was initialised with '{backend}'. This error can be "
360
- "resolved by removing '_backend' from the request."
361
- )
362
- else:
363
- params.structured_outputs._backend = backend
364
-
365
- # Request content validation
366
- if (
367
- isinstance(params.structured_outputs.choice, list)
368
- and not params.structured_outputs.choice
369
- ):
370
- # It is invalid for choice to be an empty list
371
- raise ValueError(
372
- f"Choice '{params.structured_outputs.choice}' cannot be an empty list" # noqa: E501
373
- )
374
- # Reject empty string grammar early to avoid engine-side crashes
375
- if (
376
- isinstance(params.structured_outputs.grammar, str)
377
- and params.structured_outputs.grammar.strip() == ""
378
- ):
379
- raise ValueError("structured_outputs.grammar cannot be an empty string")
380
-
381
- if backend.startswith("xgrammar"):
382
- # xgrammar with no fallback
383
- validate_xgrammar_grammar(params)
384
- elif backend.startswith("guidance"):
385
- # TODO: ideally we would have the LLTokenizer here as Lark syntax
386
- # allows <|special_token|> and similar, see
387
- # https://github.com/guidance-ai/llguidance/blob/main/docs/syntax.md#special-tokens
388
- # Without tokenizer these are disallowed in grammars.
389
- if isinstance(self.tokenizer, MistralTokenizer):
390
- raise ValueError(
391
- "Mistral tokenizer is not supported for the 'guidance' "
392
- "structured output backend. Please use ['xgrammar', 'outlines'] "
393
- "backends or tokenizer_mode='hf' instead."
394
- )
395
- validate_guidance_grammar(params, tokenizer=None)
396
- elif backend == "outlines":
397
- # outlines backend
398
- validate_structured_output_request_outlines(params)
399
- elif backend == "lm-format-enforcer":
400
- # lm format enforcer backend
401
- if isinstance(self.tokenizer, MistralTokenizer):
402
- raise ValueError(
403
- "Mistral tokenizer is not supported for the 'lm-format-enforcer' "
404
- "structured output backend. Please use ['xgrammar', 'outlines'] "
405
- "backends or tokenizer_mode='hf' instead."
406
- )
407
- validate_structured_output_request_lm_format_enforcer(params)
408
- else:
409
- # NOTE: backend must be "auto" here, because we have
410
- # checked supported_backends above.
411
- # In this mode, we set opinionated defaults based on what we think
412
- # will satisfy the most use cases without having to worry about
413
- # this setting. We include fallback behavior here, but not with any
414
- # other setting where a specific backend was specified.
415
- try:
416
- validate_xgrammar_grammar(params)
417
- params.structured_outputs._backend = "xgrammar"
418
- except ValueError:
419
- # The request either failed validation
420
- # or includes some jsonschema feature(s) that
421
- # are not supported in xgrammar.
422
-
423
- # Check if schema has features unsupported by guidance
424
- so_params = params.structured_outputs
425
- skip_guidance = False
426
- if so_params.json:
427
- if isinstance(so_params.json, str):
428
- import json
429
-
430
- schema = json.loads(so_params.json)
431
- else:
432
- schema = so_params.json
433
- skip_guidance = has_guidance_unsupported_json_features(schema)
434
-
435
- if isinstance(self.tokenizer, MistralTokenizer) or skip_guidance:
436
- # Fall back to outlines if the tokenizer is Mistral
437
- # or if schema contains features unsupported by guidance
438
- validate_structured_output_request_outlines(params)
439
- params.structured_outputs._backend = "outlines"
440
- else:
441
- # Fall back to guidance by default.
442
- validate_guidance_grammar(params, tokenizer=None)
443
- params.structured_outputs._backend = "guidance"
444
- # Remember that this backend was set automatically
445
- params.structured_outputs._backend_was_auto = True
446
-
447
- # Run post-init validation. This is also important to ensure subsequent
448
- # roundtrip serialization/deserialization won't fail.
449
- params.structured_outputs.__post_init__()
450
-
233
def _extract_singleton_mm_data(
234
self, prompt: SingletonPrompt
235
) -> MultiModalDataDict | None:
@@ -618,8 +400,10 @@ class InputProcessor:
400
prompt_token_ids, prompt_embeds
401
)
402
sampling_params.max_tokens = self.model_config.max_model_len - seq_len
403
+
404
sampling_params.update_from_generation_config(
622
- self.generation_config_fields, eos_token_id
405
+ self.generation_config_fields,
406
+ None if self.tokenizer is None else self.tokenizer.eos_token_id,
407
)
408
if self.tokenizer is not None:
409
sampling_params.update_from_tokenizer(self.tokenizer)