@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a6977dbd15
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 12:15 UTC

Simplify (and fix) passing of guided decoding backend options (#17008)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>

Harry Mellor committed Apr 29, 2025 at 20:02 UTC a6977dbd1531378456725e5cdb151c88a33df52a
17 files changed +309 -217
examples/online_serving/openai_chat_completion_structured_outputs.py
+4 -3
@@ -112,8 +112,8 @@ def extra_backend_options_completion(client: OpenAI, model: str):
112 "alan.turing@enigma.com\n")
113
114 try:
115 - # The no-fallback option forces vLLM to use xgrammar, so when it fails
116 - # you get a 400 with the reason why
115 + # The guided_decoding_disable_fallback option forces vLLM to use
116 + # xgrammar, so when it fails you get a 400 with the reason why
117 completion = client.chat.completions.create(
118 model=model,
119 messages=[{
@@ -123,7 +123,8 @@ def extra_backend_options_completion(client: OpenAI, model: str):
123 extra_body={
124 "guided_regex": r"\w+@\w+\.com\n",
125 "stop": ["\n"],
126 - "guided_decoding_backend": "xgrammar:no-fallback"
126 + "guided_decoding_backend": "xgrammar",
127 + "guided_decoding_disable_fallback": True,
128 },
129 )
130 return completion.choices[0].message.content
tests/entrypoints/llm/test_guided_generate.py
+125 -81
@@ -16,10 +16,11 @@ from vllm.sampling_params import GuidedDecodingParams, SamplingParams
16
17 MODEL_NAME = "Qwen/Qwen2.5-1.5B-Instruct"
18 GUIDED_DECODING_BACKENDS = [
19 - "outlines",
20 - "lm-format-enforcer",
21 - "xgrammar:disable-any-whitespace",
22 - "guidance:disable-any-whitespace",
19 + # (backend, disable_any_whitespace),
20 + ("outlines", False),
21 + ("lm-format-enforcer", False),
22 + ("xgrammar", True),
23 + ("guidance", True),
24 ]
25
26
@@ -36,13 +37,17 @@ def llm():
37
38
39 @pytest.mark.skip_global_cleanup
39 -@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)
40 -def test_guided_regex(sample_regex, llm, guided_decoding_backend: str):
41 - sampling_params = SamplingParams(temperature=0.8,
42 - top_p=0.95,
43 - guided_decoding=GuidedDecodingParams(
44 - regex=sample_regex,
45 - backend=guided_decoding_backend))
40 +@pytest.mark.parametrize("guided_decoding_backend,disable_any_whitespace",
41 + GUIDED_DECODING_BACKENDS)
42 +def test_guided_regex(sample_regex, llm, guided_decoding_backend: str,
43 + disable_any_whitespace: bool):
44 + sampling_params = SamplingParams(
45 + temperature=0.8,
46 + top_p=0.95,
47 + guided_decoding=GuidedDecodingParams(
48 + regex=sample_regex,
49 + backend=guided_decoding_backend,
50 + disable_any_whitespace=disable_any_whitespace))
51 outputs = llm.generate(prompts=[
52 f"Give an example IPv4 address with this regex: {sample_regex}"
53 ] * 2,
@@ -62,14 +67,18 @@ def test_guided_regex(sample_regex, llm, guided_decoding_backend: str):
67
68
69 @pytest.mark.skip_global_cleanup
65 -@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)
70 +@pytest.mark.parametrize("guided_decoding_backend,disable_any_whitespace",
71 + GUIDED_DECODING_BACKENDS)
72 def test_guided_json_completion(sample_json_schema, llm,
67 - guided_decoding_backend: str):
68 - sampling_params = SamplingParams(temperature=1.0,
69 - max_tokens=1000,
70 - guided_decoding=GuidedDecodingParams(
71 - json=sample_json_schema,
72 - backend=guided_decoding_backend))
73 + guided_decoding_backend: str,
74 + disable_any_whitespace: bool):
75 + sampling_params = SamplingParams(
76 + temperature=1.0,
77 + max_tokens=1000,
78 + guided_decoding=GuidedDecodingParams(
79 + json=sample_json_schema,
80 + backend=guided_decoding_backend,
81 + disable_any_whitespace=disable_any_whitespace))
82 outputs = llm.generate(prompts=[
83 f"Give an example JSON for an employee profile "
84 f"that fits this schema: {sample_json_schema}"
@@ -92,14 +101,18 @@ def test_guided_json_completion(sample_json_schema, llm,
101
102
103 @pytest.mark.skip_global_cleanup
95 -@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)
104 +@pytest.mark.parametrize("guided_decoding_backend,disable_any_whitespace",
105 + GUIDED_DECODING_BACKENDS)
106 def test_guided_complex_json_completion(sample_complex_json_schema, llm,
97 - guided_decoding_backend: str):
98 - sampling_params = SamplingParams(temperature=1.0,
99 - max_tokens=1000,
100 - guided_decoding=GuidedDecodingParams(
101 - json=sample_complex_json_schema,
102 - backend=guided_decoding_backend))
107 + guided_decoding_backend: str,
108 + disable_any_whitespace: bool):
109 + sampling_params = SamplingParams(
110 + temperature=1.0,
111 + max_tokens=1000,
112 + guided_decoding=GuidedDecodingParams(
113 + json=sample_complex_json_schema,
114 + backend=guided_decoding_backend,
115 + disable_any_whitespace=disable_any_whitespace))
116 outputs = llm.generate(prompts=[
117 f"Give an example JSON for an assignment grade "
118 f"that fits this schema: {sample_complex_json_schema}"
@@ -123,14 +136,18 @@ def test_guided_complex_json_completion(sample_complex_json_schema, llm,
136
137
138 @pytest.mark.skip_global_cleanup
126 -@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)
139 +@pytest.mark.parametrize("guided_decoding_backend,disable_any_whitespace",
140 + GUIDED_DECODING_BACKENDS)
141 def test_guided_definition_json_completion(sample_definition_json_schema, llm,
128 - guided_decoding_backend: str):
129 - sampling_params = SamplingParams(temperature=1.0,
130 - max_tokens=1000,
131 - guided_decoding=GuidedDecodingParams(
132 - json=sample_definition_json_schema,
133 - backend=guided_decoding_backend))
142 + guided_decoding_backend: str,
143 + disable_any_whitespace: bool):
144 + sampling_params = SamplingParams(
145 + temperature=1.0,
146 + max_tokens=1000,
147 + guided_decoding=GuidedDecodingParams(
148 + json=sample_definition_json_schema,
149 + backend=guided_decoding_backend,
150 + disable_any_whitespace=disable_any_whitespace))
151 outputs = llm.generate(prompts=[
152 f"Give an example JSON for solving 8x + 7 = -23 "
153 f"that fits this schema: {sample_definition_json_schema}"
@@ -154,14 +171,18 @@ def test_guided_definition_json_completion(sample_definition_json_schema, llm,
171
172
173 @pytest.mark.skip_global_cleanup
157 -@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)
174 +@pytest.mark.parametrize("guided_decoding_backend,disable_any_whitespace",
175 + GUIDED_DECODING_BACKENDS)
176 def test_guided_enum_json_completion(sample_enum_json_schema, llm,
159 - guided_decoding_backend: str):
160 - sampling_params = SamplingParams(temperature=1.0,
161 - max_tokens=1000,
162 - guided_decoding=GuidedDecodingParams(
163 - json=sample_enum_json_schema,
164 - backend=guided_decoding_backend))
177 + guided_decoding_backend: str,
178 + disable_any_whitespace: bool):
179 + sampling_params = SamplingParams(
180 + temperature=1.0,
181 + max_tokens=1000,
182 + guided_decoding=GuidedDecodingParams(
183 + json=sample_enum_json_schema,
184 + backend=guided_decoding_backend,
185 + disable_any_whitespace=disable_any_whitespace))
186 outputs = llm.generate(prompts=[
187 "Create a bug report JSON that fits this schema: "
188 f"{sample_enum_json_schema}. Make it for a high priority critical bug."
@@ -195,14 +216,18 @@ def test_guided_enum_json_completion(sample_enum_json_schema, llm,
216
217
218 @pytest.mark.skip_global_cleanup
198 -@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)
219 +@pytest.mark.parametrize("guided_decoding_backend,disable_any_whitespace",
220 + GUIDED_DECODING_BACKENDS)
221 def test_guided_choice_completion(sample_guided_choice, llm,
200 - guided_decoding_backend: str):
201 - sampling_params = SamplingParams(temperature=0.8,
202 - top_p=0.95,
203 - guided_decoding=GuidedDecodingParams(
204 - choice=sample_guided_choice,
205 - backend=guided_decoding_backend))
222 + guided_decoding_backend: str,
223 + disable_any_whitespace: bool):
224 + sampling_params = SamplingParams(
225 + temperature=0.8,
226 + top_p=0.95,
227 + guided_decoding=GuidedDecodingParams(
228 + choice=sample_guided_choice,
229 + backend=guided_decoding_backend,
230 + disable_any_whitespace=disable_any_whitespace))
231 outputs = llm.generate(
232 prompts="The best language for type-safe systems programming is ",
233 sampling_params=sampling_params,
@@ -221,15 +246,19 @@ def test_guided_choice_completion(sample_guided_choice, llm,
246
247
248 @pytest.mark.skip_global_cleanup
224 -@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)
249 +@pytest.mark.parametrize("guided_decoding_backend,disable_any_whitespace",
250 + GUIDED_DECODING_BACKENDS)
251 def test_guided_grammar(sample_sql_statements, llm,
226 - guided_decoding_backend: str):
227 - sampling_params = SamplingParams(temperature=0.8,
228 - top_p=0.95,
229 - max_tokens=1000,
230 - guided_decoding=GuidedDecodingParams(
231 - grammar=sample_sql_statements,
232 - backend=guided_decoding_backend))
252 + guided_decoding_backend: str,
253 + disable_any_whitespace: bool):
254 + sampling_params = SamplingParams(
255 + temperature=0.8,
256 + top_p=0.95,
257 + max_tokens=1000,
258 + guided_decoding=GuidedDecodingParams(
259 + grammar=sample_sql_statements,
260 + backend=guided_decoding_backend,
261 + disable_any_whitespace=disable_any_whitespace))
262 outputs = llm.generate(
263 prompts=("Generate a sql state that select col_1 from "
264 "table_1 where it is equals to 1"),
@@ -300,7 +329,8 @@ def test_disable_guided_decoding_fallback(sample_regex, llm):
329 top_p=0.95,
330 guided_decoding=GuidedDecodingParams(
331 json=unsupported_json,
303 - backend="xgrammar:no-fallback"))
332 + backend="xgrammar",
333 + disable_fallback=True))
334
335 with pytest.raises(
336 ValueError,
@@ -312,14 +342,18 @@ def test_disable_guided_decoding_fallback(sample_regex, llm):
342
343
344 @pytest.mark.skip_global_cleanup
315 -@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)
316 -def test_guided_json_object(llm, guided_decoding_backend: str):
317 - sampling_params = SamplingParams(temperature=1.0,
318 - max_tokens=100,
319 - n=2,
320 - guided_decoding=GuidedDecodingParams(
321 - json_object=True,
322 - backend=guided_decoding_backend))
345 +@pytest.mark.parametrize("guided_decoding_backend,disable_any_whitespace",
346 + GUIDED_DECODING_BACKENDS)
347 +def test_guided_json_object(llm, guided_decoding_backend: str,
348 + disable_any_whitespace: bool):
349 + sampling_params = SamplingParams(
350 + temperature=1.0,
351 + max_tokens=100,
352 + n=2,
353 + guided_decoding=GuidedDecodingParams(
354 + json_object=True,
355 + backend=guided_decoding_backend,
356 + disable_any_whitespace=disable_any_whitespace))
357
358 outputs = llm.generate(
359 prompts=("Generate a JSON object with curly braces for a person with "
@@ -337,7 +371,7 @@ def test_guided_json_object(llm, guided_decoding_backend: str):
371 print(generated_text)
372 assert generated_text is not None
373
340 - if 'disable-any-whitespace' in guided_decoding_backend:
374 + if disable_any_whitespace:
375 assert "\n" not in generated_text
376
377 # Parse to verify it is valid JSON
@@ -359,14 +393,18 @@ class CarDescription(BaseModel):
393
394
395 @pytest.mark.skip_global_cleanup
362 -@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)
363 -def test_guided_json_completion_with_enum(llm, guided_decoding_backend: str):
396 +@pytest.mark.parametrize("guided_decoding_backend,disable_any_whitespace",
397 + GUIDED_DECODING_BACKENDS)
398 +def test_guided_json_completion_with_enum(llm, guided_decoding_backend: str,
399 + disable_any_whitespace: bool):
400 json_schema = CarDescription.model_json_schema()
365 - sampling_params = SamplingParams(temperature=1.0,
366 - max_tokens=1000,
367 - guided_decoding=GuidedDecodingParams(
368 - json=json_schema,
369 - backend=guided_decoding_backend))
401 + sampling_params = SamplingParams(
402 + temperature=1.0,
403 + max_tokens=1000,
404 + guided_decoding=GuidedDecodingParams(
405 + json=json_schema,
406 + backend=guided_decoding_backend,
407 + disable_any_whitespace=disable_any_whitespace))
408 outputs = llm.generate(
409 prompts="Generate a JSON with the brand, model and car_type of"
410 "the most iconic car from the 90's",
@@ -387,9 +425,10 @@ def test_guided_json_completion_with_enum(llm, guided_decoding_backend: str):
425
426
427 @pytest.mark.skip_global_cleanup
390 -@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)
391 -def test_guided_number_range_json_completion(llm,
392 - guided_decoding_backend: str):
428 +@pytest.mark.parametrize("guided_decoding_backend,disable_any_whitespace",
429 + GUIDED_DECODING_BACKENDS)
430 +def test_guided_number_range_json_completion(llm, guided_decoding_backend: str,
431 + disable_any_whitespace: bool):
432 sample_output_schema = {
433 "type": "object",
434 "properties": {
@@ -413,8 +452,10 @@ def test_guided_number_range_json_completion(llm,
452 sampling_params = SamplingParams(
453 temperature=1.0,
454 max_tokens=1000,
416 - guided_decoding=GuidedDecodingParams(json=sample_output_schema,
417 - backend=guided_decoding_backend),
455 + guided_decoding=GuidedDecodingParams(
456 + json=sample_output_schema,
457 + backend=guided_decoding_backend,
458 + disable_any_whitespace=disable_any_whitespace),
459 )
460 outputs = llm.generate(
461 prompts=[
@@ -466,8 +507,12 @@ def test_guidance_no_additional_properties(llm):
507 "large JSON object with key-value pairs a1=b1, a2=b2, ..., a20=b20"
508 "<|im_end|>\n<|im_start|>assistant\n")
509
469 - def generate_with_backend(backend):
470 - guided_params = GuidedDecodingParams(json=schema, backend=backend)
510 + def generate_with_backend(backend, disable_additional_properties):
511 + guided_params = GuidedDecodingParams(
512 + json=schema,
513 + backend=backend,
514 + disable_any_whitespace=True,
515 + disable_additional_properties=disable_additional_properties)
516 sampling_params = SamplingParams(temperature=0,
517 max_tokens=256,
518 guided_decoding=guided_params)
@@ -481,7 +526,7 @@ def test_guidance_no_additional_properties(llm):
526 jsonschema.validate(instance=parsed_json, schema=schema)
527 return parsed_json
528
484 - base_generated = generate_with_backend('guidance:disable-any-whitespace')
529 + base_generated = generate_with_backend("guidance", False)
530 assert "a1" in base_generated
531 assert "a2" in base_generated
532 assert "a3" in base_generated
@@ -490,8 +535,7 @@ def test_guidance_no_additional_properties(llm):
535 assert "a5" in base_generated
536 assert "a6" in base_generated
537
493 - generated = generate_with_backend(
494 - 'guidance:no-additional-properties,disable-any-whitespace')
538 + generated = generate_with_backend("guidance", True)
539 assert "a1" in generated
540 assert "a2" in generated
541 assert "a3" in generated
tests/model_executor/test_guided_processors.py
+9 -6
@@ -202,12 +202,15 @@ def test_multiple_guided_options_not_allowed(sample_json_schema, sample_regex):
202
203 def test_guided_decoding_backend_options():
204 """Test backend-specific options"""
205 - params = GuidedDecodingParams(
206 - backend="xgrammar:option-1,option-2,option-3")
207 - assert params.backend_options() == ["option-1", "option-2", "option-3"]
208 -
209 - no_fallback = GuidedDecodingParams(backend="xgrammar:option-1,no-fallback")
210 - assert no_fallback.no_fallback()
205 + with pytest.warns(DeprecationWarning):
206 + guided_decoding_params = GuidedDecodingParams(
207 + backend=
208 + "xgrammar:no-fallback,disable-any-whitespace,no-additional-properties"
209 + )
210 + assert guided_decoding_params.backend == "xgrammar"
211 + assert guided_decoding_params.disable_fallback
212 + assert guided_decoding_params.disable_any_whitespace
213 + assert guided_decoding_params.disable_additional_properties
214
215
216 def test_pickle_xgrammar_tokenizer_data():
tests/v1/entrypoints/llm/test_struct_output_generate.py
+16 -15
@@ -17,15 +17,12 @@ from vllm.platforms import current_platform
17 from vllm.sampling_params import GuidedDecodingParams, SamplingParams
18
19 PARAMS_MODELS_BACKENDS_TOKENIZER_MODE = [
20 - ("mistralai/Ministral-8B-Instruct-2410", "xgrammar:disable-any-whitespace",
21 - "auto"),
22 - ("mistralai/Ministral-8B-Instruct-2410", "guidance:disable-any-whitespace",
23 - "auto"),
24 - ("mistralai/Ministral-8B-Instruct-2410", "xgrammar:disable-any-whitespace",
25 - "mistral"),
26 - ("Qwen/Qwen2.5-1.5B-Instruct", "xgrammar:disable-any-whitespace", "auto"),
20 + ("mistralai/Ministral-8B-Instruct-2410", "xgrammar", "auto"),
21 + ("mistralai/Ministral-8B-Instruct-2410", "guidance", "auto"),
22 + ("mistralai/Ministral-8B-Instruct-2410", "xgrammar", "mistral"),
23 + ("Qwen/Qwen2.5-1.5B-Instruct", "xgrammar", "auto"),
24 #FIXME: This test is flaky on CI thus disabled
28 - #("Qwen/Qwen2.5-1.5B-Instruct", "guidance:disable-any-whitespace", "auto"),
25 + #("Qwen/Qwen2.5-1.5B-Instruct", "guidance", "auto"),
26 ]
27
28 PARAMS_MODELS_TOKENIZER_MODE = [
@@ -73,6 +70,7 @@ def test_structured_output(
70 enforce_eager=enforce_eager,
71 max_model_len=1024,
72 guided_decoding_backend=guided_decoding_backend,
73 + guided_decoding_disable_any_whitespace=True,
74 tokenizer_mode=tokenizer_mode)
75
76 #
@@ -98,8 +96,7 @@ def test_structured_output(
96
97 generated_text = output.outputs[0].text
98 assert generated_text is not None
101 - if 'disable-any-whitespace' in guided_decoding_backend:
102 - assert "\n" not in generated_text
99 + assert "\n" not in generated_text
100 print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
101 output_json = json.loads(generated_text)
102 jsonschema.validate(instance=output_json, schema=sample_json_schema)
@@ -520,10 +517,11 @@ def test_structured_output_auto_mode(
517 def test_guidance_no_additional_properties(monkeypatch: pytest.MonkeyPatch):
518 monkeypatch.setenv("VLLM_USE_V1", "1")
519
523 - backend = 'guidance:no-additional-properties,disable-any-whitespace'
520 llm = LLM(model="Qwen/Qwen2.5-1.5B-Instruct",
521 max_model_len=1024,
526 - guided_decoding_backend=backend)
522 + guided_decoding_backend="guidance",
523 + guided_decoding_disable_any_whitespace=True,
524 + guided_decoding_disable_additional_properties=True)
525
526 schema = {
527 'type': 'object',
@@ -548,7 +546,11 @@ def test_guidance_no_additional_properties(monkeypatch: pytest.MonkeyPatch):
546 "<|im_end|>\n<|im_start|>assistant\n")
547
548 def generate_with_backend(backend):
551 - guided_params = GuidedDecodingParams(json=schema, backend=backend)
549 + guided_params = GuidedDecodingParams(
550 + json=schema,
551 + backend=backend,
552 + disable_any_whitespace=True,
553 + disable_additional_properties=True)
554 sampling_params = SamplingParams(temperature=0,
555 max_tokens=256,
556 guided_decoding=guided_params)
@@ -562,8 +564,7 @@ def test_guidance_no_additional_properties(monkeypatch: pytest.MonkeyPatch):
564 jsonschema.validate(instance=parsed_json, schema=schema)
565 return parsed_json
566
565 - generated = generate_with_backend(
566 - 'guidance:no-additional-properties,disable-any-whitespace')
567 + generated = generate_with_backend("guidance")
568 assert "a1" in generated
569 assert "a2" in generated
570 assert "a3" in generated
tests/v1/test_oracle.py
+2 -1
@@ -57,7 +57,8 @@ def test_unsupported_configs(monkeypatch):
57 with pytest.raises(NotImplementedError):
58 AsyncEngineArgs(
59 model=MODEL,
60 - guided_decoding_backend="lm-format-enforcer:no-fallback",
60 + guided_decoding_backend="lm-format-enforcer",
61 + guided_decoding_disable_fallback=True,
62 ).create_engine_config()
63
64 with pytest.raises(NotImplementedError):
vllm/config.py
+60 -10
@@ -17,12 +17,14 @@ from dataclasses import (MISSING, dataclass, field, fields, is_dataclass,
17 from importlib.util import find_spec
18 from pathlib import Path
19 from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Final, Literal,
20 - Optional, Protocol, TypeVar, Union, get_args, get_origin)
20 + Optional, Protocol, TypeVar, Union, cast, get_args,
21 + get_origin)
22
23 import torch
24 from pydantic import BaseModel, Field, PrivateAttr
25 from torch.distributed import ProcessGroup, ReduceOp
26 from transformers import PretrainedConfig
27 +from typing_extensions import deprecated
28
29 import vllm.envs as envs
30 from vllm.compilation.inductor_pass import CallableInductorPass, InductorPass
@@ -32,7 +34,6 @@ from vllm.model_executor.layers.quantization import (QUANTIZATION_METHODS,
34 get_quantization_config)
35 from vllm.model_executor.models import ModelRegistry
36 from vllm.platforms import CpuArchEnum, current_platform
35 -from vllm.sampling_params import GuidedDecodingParams
37 from vllm.tracing import is_otel_available, otel_import_error_traceback
38 from vllm.transformers_utils.config import (
39 ConfigFormat, get_config, get_hf_image_processor_config,
@@ -344,7 +345,7 @@ class ModelConfig:
345 def __init__(
346 self,
347 model: str,
347 - task: Union[TaskOption, Literal["draft"]],
348 + task: Literal[TaskOption, Literal["draft"]],
349 tokenizer: str,
350 tokenizer_mode: str,
351 trust_remote_code: bool,
@@ -701,7 +702,7 @@ class ModelConfig:
702
703 def _resolve_task(
704 self,
704 - task_option: Union[TaskOption, Literal["draft"]],
705 + task_option: Literal[TaskOption, Literal["draft"]],
706 ) -> tuple[set[_ResolvedTask], _ResolvedTask]:
707 if task_option == "draft":
708 return {"draft"}, "draft"
@@ -3185,13 +3186,36 @@ GuidedDecodingBackend = Literal[GuidedDecodingBackendV0,
3186 class DecodingConfig:
3187 """Dataclass which contains the decoding strategy of the engine."""
3188
3188 - guided_decoding_backend: GuidedDecodingBackend = \
3189 - "auto" if envs.VLLM_USE_V1 else "xgrammar"
3189 + @property
3190 + @deprecated(
3191 + "`guided_decoding_backend` is deprecated and has been renamed to "
3192 + "`backend`. This will be removed in v0.10.0. Please use the "
3193 + "`backend` argument instead.")
3194 + def guided_decoding_backend(self) -> GuidedDecodingBackend:
3195 + return self.backend
3196 +
3197 + @guided_decoding_backend.setter
3198 + def guided_decoding_backend(self, value: GuidedDecodingBackend):
3199 + self.backend = value
3200 +
3201 + backend: GuidedDecodingBackend = "auto" if envs.VLLM_USE_V1 else "xgrammar"
3202 """Which engine will be used for guided decoding (JSON schema / regex etc)
3203 by default. With "auto", we will make opinionated choices based on request
3204 contents and what the backend libraries currently support, so the behavior
3205 is subject to change in each release."""
3206
3207 + disable_fallback: bool = False
3208 + """If `True`, vLLM will not fallback to a different backend on error."""
3209 +
3210 + disable_any_whitespace: bool = False
3211 + """If `True`, the model will not generate any whitespace during guided
3212 + decoding. This is only supported for xgrammar and guidance backends."""
3213 +
3214 + disable_additional_properties: bool = False
3215 + """If `True`, the `guidance` backend will not use `additionalProperties`
3216 + in the JSON schema. This is only supported for the `guidance` backend and
3217 + is used to better align its behaviour with `outlines` and `xgrammar`."""
3218 +
3219 reasoning_backend: Optional[str] = None
3220 """Select the reasoning parser depending on the model that you're using.
3221 This is used to parse the reasoning content into OpenAI API format.
@@ -3217,15 +3241,41 @@ class DecodingConfig:
3241 return hash_str
3242
3243 def __post_init__(self):
3220 - backend = GuidedDecodingParams(
3221 - backend=self.guided_decoding_backend).backend_name
3244 + if ":" in self.backend:
3245 + self._extract_backend_options()
3246 +
3247 if envs.VLLM_USE_V1:
3248 valid_guided_backends = get_args(GuidedDecodingBackendV1)
3249 else:
3250 valid_guided_backends = get_args(GuidedDecodingBackendV0)
3226 - if backend not in valid_guided_backends:
3227 - raise ValueError(f"Invalid guided_decoding_backend '{backend}',"
3251 + if self.backend not in valid_guided_backends:
3252 + raise ValueError(f"Invalid backend '{self.backend}',"
3253 f" must be one of {valid_guided_backends}")
3254 + if (self.disable_any_whitespace
3255 + and self.backend not in ("xgrammar", "guidance")):
3256 + raise ValueError("disable_any_whitespace is only supported for "
3257 + "xgrammar and guidance backends.")
3258 + if (self.disable_additional_properties and self.backend != "guidance"):
3259 + raise ValueError("disable_additional_properties is only supported "
3260 + "for the guidance backend.")
3261 +
3262 + @deprecated(
3263 + "Passing guided decoding backend options inside backend in the format "
3264 + "'backend:...' is deprecated. This will be removed in v0.10.0. Please "
3265 + "use the dedicated arguments '--disable-fallback', "
3266 + "'--disable-any-whitespace' and '--disable-additional-properties' "
3267 + "instead.")
3268 + def _extract_backend_options(self):
3269 + """Extract backend options from the backend string."""
3270 + backend, options = self.backend.split(":")
3271 + self.backend = cast(GuidedDecodingBackend, backend)
3272 + options_set = set(options.strip().split(","))
3273 + if "no-fallback" in options_set:
3274 + self.disable_fallback = True
3275 + if "disable-any-whitespace" in options_set:
3276 + self.disable_any_whitespace = True
3277 + if "no-additional-properties" in options_set:
3278 + self.disable_additional_properties = True
3279
3280
3281 @dataclass
vllm/engine/arg_utils.py
+26 -10
@@ -18,9 +18,9 @@ from vllm import version
18 from vllm.config import (BlockSize, CacheConfig, CacheDType, CompilationConfig,
19 ConfigFormat, ConfigType, DecodingConfig, Device,
20 DeviceConfig, DistributedExecutorBackend,
21 - GuidedDecodingBackendV1, HfOverrides,
22 - KVTransferConfig, LoadConfig, LoadFormat, LoRAConfig,
23 - ModelConfig, ModelImpl, MultiModalConfig,
21 + GuidedDecodingBackend, GuidedDecodingBackendV1,
22 + HfOverrides, KVTransferConfig, LoadConfig, LoadFormat,
23 + LoRAConfig, ModelConfig, ModelImpl, MultiModalConfig,
24 ObservabilityConfig, ParallelConfig, PoolerConfig,
25 PrefixCachingHashAlgo, PromptAdapterConfig,
26 SchedulerConfig, SchedulerPolicy, SpeculativeConfig,
@@ -317,7 +317,12 @@ class EngineArgs:
317 bool] = SchedulerConfig.enable_chunked_prefill
318 disable_chunked_mm_input: bool = SchedulerConfig.disable_chunked_mm_input
319
320 - guided_decoding_backend: str = DecodingConfig.guided_decoding_backend
320 + guided_decoding_backend: GuidedDecodingBackend = DecodingConfig.backend
321 + guided_decoding_disable_fallback: bool = DecodingConfig.disable_fallback
322 + guided_decoding_disable_any_whitespace: bool = \
323 + DecodingConfig.disable_any_whitespace
324 + guided_decoding_disable_additional_properties: bool = \
325 + DecodingConfig.disable_additional_properties
326 logits_processor_pattern: Optional[str] = None
327
328 speculative_config: Optional[Dict[str, Any]] = None
@@ -498,9 +503,17 @@ class EngineArgs:
503 title="DecodingConfig",
504 description=DecodingConfig.__doc__,
505 )
506 + guided_decoding_group.add_argument("--guided-decoding-backend",
507 + **guided_decoding_kwargs["backend"])
508 guided_decoding_group.add_argument(
502 - '--guided-decoding-backend',
503 - **guided_decoding_kwargs["guided_decoding_backend"])
509 + "--guided-decoding-disable-fallback",
510 + **guided_decoding_kwargs["disable_fallback"])
511 + guided_decoding_group.add_argument(
512 + "--guided-decoding-disable-any-whitespace",
513 + **guided_decoding_kwargs["disable_any_whitespace"])
514 + guided_decoding_group.add_argument(
515 + "--guided-decoding-disable-additional-properties",
516 + **guided_decoding_kwargs["disable_additional_properties"])
517 guided_decoding_group.add_argument(
518 "--reasoning-parser",
519 # This choices is a special case because it's not static
@@ -1244,7 +1257,11 @@ class EngineArgs:
1257 if self.enable_prompt_adapter else None
1258
1259 decoding_config = DecodingConfig(
1247 - guided_decoding_backend=self.guided_decoding_backend,
1260 + backend=self.guided_decoding_backend,
1261 + disable_fallback=self.guided_decoding_disable_fallback,
1262 + disable_any_whitespace=self.guided_decoding_disable_any_whitespace,
1263 + disable_additional_properties=\
1264 + self.guided_decoding_disable_additional_properties,
1265 reasoning_backend=self.reasoning_parser
1266 if self.enable_reasoning else None,
1267 )
@@ -1335,9 +1352,8 @@ class EngineArgs:
1352 recommend_to_remove=True)
1353 return False
1354
1338 - # remove backend options when doing this check
1339 - if self.guided_decoding_backend.split(':')[0] \
1340 - not in get_args(GuidedDecodingBackendV1):
1355 + if self.guided_decoding_backend not in get_args(
1356 + GuidedDecodingBackendV1):
1357 _raise_or_fallback(
1358 feature_name=
1359 f"--guided-decoding-backend={self.guided_decoding_backend}",
vllm/engine/llm_engine.py
+1 -1
@@ -2091,7 +2091,7 @@ class LLMEngine:
2091
2092 tokenizer = self.get_tokenizer(lora_request=lora_request)
2093 guided_decoding.backend = guided_decoding.backend or \
2094 - self.decoding_config.guided_decoding_backend
2094 + self.decoding_config.backend
2095
2096 if self.decoding_config.reasoning_backend is not None:
2097 logger.debug("Building with reasoning backend %s",
vllm/engine/multiprocessing/client.py
+2 -2
@@ -615,9 +615,9 @@ class MQLLMEngineClient(EngineClient):
615 build_guided_decoding_logits_processor_async(
616 sampling_params=params,
617 tokenizer=await self.get_tokenizer(lora_request),
618 - default_guided_backend=(self.decoding_config.guided_decoding_backend
618 + default_guided_backend=(self.decoding_config.backend
619 if self.decoding_config
620 - else DecodingConfig.guided_decoding_backend),
620 + else DecodingConfig.backend),
621 model_config=self.model_config,
622 reasoning_backend=self.decoding_config.reasoning_backend,
623 )
vllm/model_executor/guided_decoding/__init__.py
+12 -12
@@ -26,8 +26,8 @@ def maybe_backend_fallback(
26 def fallback_or_error(guided_params: GuidedDecodingParams, message: str,
27 fallback: str) -> None:
28 """Change the backend to the specified fallback with a warning log,
29 - or raise a ValueError if the `no-fallback` option is specified."""
30 - if guided_params.no_fallback():
29 + or raise a ValueError if the `disable_fallback` option is specified."""
30 + if guided_params.disable_fallback:
31 raise ValueError(message)
32
33 logger.warning("%s Falling back to use %s instead.", message, fallback)
@@ -40,7 +40,7 @@ def maybe_backend_fallback(
40 guided_params.backend = "xgrammar"
41
42 # lm-format-enforce doesn't support grammar, fallback to xgrammar
43 - if guided_params.backend_name == "lm-format-enforcer":
43 + if guided_params.backend == "lm-format-enforcer":
44 if guided_params.grammar is not None:
45 fallback_or_error(
46 guided_params,
@@ -55,7 +55,7 @@ def maybe_backend_fallback(
55 "lm-format-enforcer does not support advanced JSON schema "
56 "features like patterns or numeric ranges.", "outlines")
57
58 - if guided_params.backend_name == "xgrammar":
58 + if guided_params.backend == "xgrammar":
59 from vllm.model_executor.guided_decoding.xgrammar_decoding import (
60 xgr_installed)
61
@@ -87,7 +87,7 @@ def maybe_backend_fallback(
87 guided_params,
88 "xgrammar module cannot be imported successfully.", "outlines")
89
90 - if (guided_params.backend_name == "outlines"
90 + if (guided_params.backend == "outlines"
91 and guided_params.json_object is not None):
92 # outlines doesn't support json_object, fallback to guidance
93 fallback_or_error(guided_params,
@@ -111,7 +111,7 @@ async def get_guided_decoding_logits_processor(
111 guided_params = maybe_backend_fallback(guided_params)
112
113 # CFG grammar not supported by LMFE, so we use outlines instead
114 - if guided_params.backend_name == 'outlines':
114 + if guided_params.backend == 'outlines':
115 # NOTE: lazy import outlines to avoid https://github.com/vllm-project/vllm/issues/4193
116 from vllm.model_executor.guided_decoding.outlines_decoding import ( # noqa
117 get_outlines_guided_decoding_logits_processor)
@@ -122,12 +122,12 @@ async def get_guided_decoding_logits_processor(
122 get_local_lm_format_enforcer_guided_decoding_logits_processor)
123 return get_local_lm_format_enforcer_guided_decoding_logits_processor(
124 guided_params, tokenizer)
125 - if guided_params.backend_name == 'xgrammar':
125 + if guided_params.backend == 'xgrammar':
126 from vllm.model_executor.guided_decoding.xgrammar_decoding import ( # noqa
127 get_local_xgrammar_guided_decoding_logits_processor)
128 return get_local_xgrammar_guided_decoding_logits_processor(
129 guided_params, tokenizer, model_config, reasoner)
130 - if guided_params.backend_name == 'guidance':
130 + if guided_params.backend == 'guidance':
131 from vllm.model_executor.guided_decoding.guidance_decoding import (
132 get_local_guidance_guided_decoding_logits_processor)
133 return get_local_guidance_guided_decoding_logits_processor(
@@ -152,23 +152,23 @@ def get_local_guided_decoding_logits_processor(
152 reasoner = reasoner_class(tokenizer)
153
154 # CFG grammar not supported by LMFE, so we use outlines instead
155 - if guided_params.backend_name == 'outlines':
155 + if guided_params.backend == 'outlines':
156 # NOTE: lazy import outlines to avoid https://github.com/vllm-project/vllm/issues/4193
157 from vllm.model_executor.guided_decoding.outlines_decoding import ( # noqa
158 get_local_outlines_guided_decoding_logits_processor)
159 return get_local_outlines_guided_decoding_logits_processor(
160 guided_params, tokenizer, reasoner)
161 - if guided_params.backend_name == 'lm-format-enforcer':
161 + if guided_params.backend == 'lm-format-enforcer':
162 from vllm.model_executor.guided_decoding.lm_format_enforcer_decoding import ( # noqa
163 get_local_lm_format_enforcer_guided_decoding_logits_processor)
164 return get_local_lm_format_enforcer_guided_decoding_logits_processor(
165 guided_params, tokenizer)
166 - if guided_params.backend_name == 'xgrammar':
166 + if guided_params.backend == 'xgrammar':
167 from vllm.model_executor.guided_decoding.xgrammar_decoding import ( # noqa
168 get_local_xgrammar_guided_decoding_logits_processor)
169 return get_local_xgrammar_guided_decoding_logits_processor(
170 guided_params, tokenizer, model_config, reasoner)
171 - if guided_params.backend_name == 'guidance':
171 + if guided_params.backend == 'guidance':
172 from vllm.model_executor.guided_decoding.guidance_decoding import (
173 get_local_guidance_guided_decoding_logits_processor)
174 return get_local_guidance_guided_decoding_logits_processor(
vllm/model_executor/guided_decoding/guidance_decoding.py
+2 -3
@@ -21,13 +21,12 @@ def get_local_guidance_guided_decoding_logits_processor(
21 """
22
23 grm = ""
24 - any_whitespace = 'disable-any-whitespace' not in \
25 - guided_params.backend_options()
24 + any_whitespace = not guided_params.disable_any_whitespace
25 if (guide_json := guided_params.json) is not None:
26 # Optionally set additionalProperties to False at the top-level
27 # By default, other backends do not allow additional top-level
28 # properties, so this makes guidance more similar to other backends
30 - if 'no-additional-properties' in guided_params.backend_options():
29 + if guided_params.disable_additional_properties:
30 if not isinstance(guide_json, str):
31 guide_json = json.dumps(guide_json)
32 guide_json = process_for_additional_properties(guide_json)
vllm/model_executor/guided_decoding/xgrammar_decoding.py
+5 -7
@@ -175,8 +175,7 @@ class GrammarConfig:
175 else:
176 json_str = guided_params.json
177
178 - any_whitespace = 'disable-any-whitespace' not in \
179 - guided_params.backend_options()
178 + any_whitespace = not guided_params.disable_any_whitespace
179
180 # Check and log if model with xgrammar and whitespace have history
181 # of runaway generation of whitespaces.
@@ -191,11 +190,10 @@ class GrammarConfig:
190 model_with_warn = 'Qwen'
191
192 if model_with_warn is not None and any_whitespace:
194 - msg = (f"{model_with_warn} "
195 - f"model detected, consider set "
196 - f"`guided_backend=xgrammar:disable-any-whitespace` "
197 - f"to prevent runaway generation of whitespaces.")
198 - logger.info_once(msg)
193 + logger.info_once(
194 + "%s model detected, consider setting "
195 + "`disable_any_whitespace` to prevent runaway generation "
196 + "of whitespaces.", model_with_warn)
197 # Validate the schema and raise ValueError here if it is invalid.
198 # This is to avoid exceptions in model execution, which will crash
199 # the engine worker process.
vllm/sampling_params.py
+26 -30
@@ -8,6 +8,7 @@ from typing import Annotated, Any, Optional, Union
8
9 import msgspec
10 from pydantic import BaseModel
11 +from typing_extensions import deprecated
12
13 from vllm.logger import init_logger
14 from vllm.logits_process import LogitsProcessor
@@ -37,6 +38,10 @@ class GuidedDecodingParams:
38 json_object: Optional[bool] = None
39 """These are other options that can be set"""
40 backend: Optional[str] = None
41 + backend_was_auto: bool = False
42 + disable_fallback: bool = False
43 + disable_any_whitespace: bool = False
44 + disable_additional_properties: bool = False
45 whitespace_pattern: Optional[str] = None
46 structural_tag: Optional[str] = None
47
@@ -68,36 +73,6 @@ class GuidedDecodingParams:
73 structural_tag=structural_tag,
74 )
75
71 - @property
72 - def backend_name(self) -> str:
73 - """Return the backend name without any options.
74 -
75 - For example if the backend is "xgrammar:no-fallback", returns "xgrammar"
76 - """
77 - return (self.backend or "").split(":")[0]
78 -
79 - def backend_options(self) -> list[str]:
80 - """Return the backend options as a list of strings."""
81 - if not self.backend or ":" not in self.backend:
82 - return []
83 - return self.backend.split(":")[1].split(",")
84 -
85 - def add_option(self, opt_name: str) -> None:
86 - """Adds an option to the backend options."""
87 - if not self.backend:
88 - self.backend = f":{opt_name}"
89 - elif ":" not in self.backend:
90 - self.backend += f":{opt_name}"
91 - else:
92 - options = set(self.backend_options())
93 - options.add(opt_name)
94 - self.backend = f"{self.backend_name}:{','.join(sorted(options))}"
95 -
96 - def no_fallback(self) -> bool:
97 - """Returns True if the "no-fallback" option is supplied for the guided
98 - decoding backend"""
99 - return "no-fallback" in self.backend_options()
100 -
76 def __post_init__(self):
77 """Validate that some fields are mutually exclusive."""
78 guide_count = sum([
@@ -109,6 +84,27 @@ class GuidedDecodingParams:
84 "You can only use one kind of guided decoding but multiple are "
85 f"specified: {self.__dict__}")
86
87 + if self.backend is not None and ":" in self.backend:
88 + self._extract_backend_options()
89 +
90 + @deprecated(
91 + "Passing guided decoding backend options inside backend in the format "
92 + "'backend:...' is deprecated. This will be removed in v0.10.0. Please "
93 + "use the dedicated arguments '--disable-fallback', "
94 + "'--disable-any-whitespace' and '--disable-additional-properties' "
95 + "instead.")
96 + def _extract_backend_options(self):
97 + """Extract backend options from the backend string."""
98 + assert isinstance(self.backend, str)
99 + self.backend, options = self.backend.split(":")
100 + options_set = set(options.strip().split(","))
101 + if "no-fallback" in options_set:
102 + self.disable_fallback = True
103 + if "disable-any-whitespace" in options_set:
104 + self.disable_any_whitespace = True
105 + if "no-additional-properties" in options_set:
106 + self.disable_additional_properties = True
107 +
108
109 class RequestOutputKind(Enum):
110 # Return entire output so far in every RequestOutput
vllm/v1/engine/processor.py
+4 -4
@@ -144,7 +144,7 @@ class Processor:
144 if not params.guided_decoding or not self.decoding_config:
145 return
146
147 - engine_level_backend = self.decoding_config.guided_decoding_backend
147 + engine_level_backend = self.decoding_config.backend
148 if params.guided_decoding.backend:
149 # Request-level backend selection is not supported in V1.
150 # The values may differ if `params` is reused and was set
@@ -152,8 +152,8 @@ class Processor:
152 # request. We remember that it was set as a result of `auto`
153 # using the `_auto` option set on the backend in the params.
154 if (params.guided_decoding.backend != engine_level_backend
155 - and not (engine_level_backend == "auto" and "_auto"
156 - in params.guided_decoding.backend_options())):
155 + and not (engine_level_backend == "auto"
156 + and params.guided_decoding.backend_was_auto)):
157 raise ValueError(
158 "Request-level structured output backend selection is no "
159 "longer supported. The request specified "
@@ -189,7 +189,7 @@ class Processor:
189 # are not supported in xgrammar. Fall back to guidance.
190 params.guided_decoding.backend = "guidance"
191 # Remember that this backend was set automatically
192 - params.guided_decoding.add_option("_auto")
192 + params.guided_decoding.backend_was_auto = True
193
194 def process_inputs(
195 self,
vllm/v1/structured_output/__init__.py
+4 -4
@@ -45,17 +45,17 @@ class StructuredOutputManager:
45 # NOTE: We only support a single backend. We do NOT support different
46 # backends on a per-request basis in V1 (for now, anyway...).
47 if self.backend is None:
48 - backend_name = request.sampling_params.guided_decoding.backend_name
49 - if backend_name == "xgrammar":
48 + backend = request.sampling_params.guided_decoding.backend
49 + if backend == "xgrammar":
50 from vllm.v1.structured_output.backend_xgrammar import (
51 XgrammarBackend)
52
53 self.backend = XgrammarBackend(self.vllm_config)
54 - elif backend_name == "guidance":
54 + elif backend == "guidance":
55 self.backend = GuidanceBackend(self.vllm_config)
56 else:
57 raise ValueError(
58 - f"Unsupported structured output backend: {backend_name}")
58 + f"Unsupported structured output backend: {backend}")
59
60 grammar = self.executor.submit(self._async_create_grammar, request)
61 request.structured_output_request.grammar = grammar # type: ignore[assignment]
vllm/v1/structured_output/backend_guidance.py
+8 -17
@@ -10,7 +10,7 @@ import torch
10
11 from vllm.config import VllmConfig
12 from vllm.logger import init_logger
13 -from vllm.sampling_params import GuidedDecodingParams, SamplingParams
13 +from vllm.sampling_params import SamplingParams
14 from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
15 from vllm.utils import LazyLoader
16 from vllm.v1.structured_output.backend_types import (StructuredOutputBackend,
@@ -65,19 +65,10 @@ class GuidanceBackend(StructuredOutputBackend):
65 self.vllm_config = vllm_config
66 self.vocab_size = vllm_config.model_config.get_vocab_size()
67
68 - self.disable_any_whitespace = False
69 - self.no_additional_properties = False
70 - backend_options = GuidedDecodingParams(
71 - backend=vllm_config.decoding_config.guided_decoding_backend
72 - ).backend_options()
73 - for option in backend_options:
74 - if option == "disable-any-whitespace":
75 - self.disable_any_whitespace = True
76 - elif option == "no-additional-properties":
77 - self.no_additional_properties = True
78 - else:
79 - raise ValueError(
80 - f"Unsupported option for the guidance backend: {option}")
68 + self.disable_any_whitespace = \
69 + vllm_config.decoding_config.disable_any_whitespace
70 + self.disable_additional_properties = \
71 + vllm_config.decoding_config.disable_additional_properties
72
73 tokenizer = tokenizer_group.get_lora_tokenizer(None)
74 self.ll_tokenizer = llguidance_hf.from_tokenizer(
@@ -87,7 +78,7 @@ class GuidanceBackend(StructuredOutputBackend):
78 grammar_spec: str) -> StructuredOutputGrammar:
79 self.serialized_grammar = serialize_guidance_grammar(
80 request_type, grammar_spec, self.disable_any_whitespace,
90 - self.no_additional_properties)
81 + self.disable_additional_properties)
82
83 ll_matcher = llguidance.LLMatcher(
84 self.ll_tokenizer,
@@ -171,11 +162,11 @@ def serialize_guidance_grammar(
162 request_type: StructuredOutputOptions,
163 grammar_spec: Union[str, dict[str, Any]],
164 disable_any_whitespace: bool = False,
174 - no_additional_properties: bool = False,
165 + disable_additional_properties: bool = False,
166 ) -> str:
167
168 def _process_schema(grammar_spec: Union[str, dict[str, Any]], ) -> str:
178 - if no_additional_properties:
169 + if disable_additional_properties:
170 grammar_spec = process_for_additional_properties(grammar_spec)
171 return llguidance.LLMatcher.grammar_from_json_schema(
172 grammar_spec,
vllm/v1/structured_output/backend_xgrammar.py
+3 -11
@@ -9,7 +9,7 @@ import torch
9 import vllm.envs
10 from vllm.config import VllmConfig
11 from vllm.logger import init_logger
12 -from vllm.sampling_params import GuidedDecodingParams, SamplingParams
12 +from vllm.sampling_params import SamplingParams
13 from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
14 from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer
15 from vllm.utils import LazyLoader
@@ -37,16 +37,8 @@ class XgrammarBackend(StructuredOutputBackend):
37 scheduler_config=vllm_config.scheduler_config,
38 lora_config=vllm_config.lora_config) # type: ignore[arg-type]
39
40 - self.disable_any_whitespace = False
41 - backend_options = GuidedDecodingParams(
42 - backend=vllm_config.decoding_config.guided_decoding_backend
43 - ).backend_options()
44 - for option in backend_options:
45 - if option == "disable-any-whitespace":
46 - self.disable_any_whitespace = True
47 - else:
48 - raise ValueError(
49 - f"Unsupported option for the xgrammar backend: {option}")
40 + self.disable_any_whitespace = \
41 + vllm_config.decoding_config.disable_any_whitespace
42
43 tokenizer = tokenizer_group.get_lora_tokenizer(None)
44 self.vocab_size = vllm_config.model_config.get_vocab_size()