A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 22:15 UTC
[Frontend] Add backend-specific options for guided decoding (#13505)
Signed-off-by: Joe Runde <Joseph.Runde@ibm.com>
Joe Runde committed
Feb 20, 2025 at 13:07 UTC
bfbc0b32c6dbea5aea6b14724acaca515d23a28a
8 files changed
+123
-42
docs/source/features/structured_outputs.md
+1
-1
@@ -16,7 +16,7 @@ The following parameters are supported, which must be added as extra parameters:
16
- `guided_json`: the output will follow the JSON schema.
17
- `guided_grammar`: the output will follow the context free grammar.
18
- `guided_whitespace_pattern`: used to override the default whitespace pattern for guided json decoding.
19
-- `guided_decoding_backend`: used to select the guided decoding backend to use.
19
+- `guided_decoding_backend`: used to select the guided decoding backend to use. Additional backend-specific options can be supplied in a comma separated list following a colon after the backend name. For example `"xgrammar:no-fallback"` will not allow vLLM to fallback to a different backend on error.
20
21
You can see the complete list of supported parameters on the [OpenAI-Compatible Server](#openai-compatible-server)page.
22
examples/online_serving/openai_chat_completion_structured_outputs.py
+24
-1
@@ -2,7 +2,7 @@
2
3
from enum import Enum
4
5
-from openai import OpenAI
5
+from openai import BadRequestError, OpenAI
6
from pydantic import BaseModel
7
8
client = OpenAI(
@@ -94,3 +94,26 @@ completion = client.chat.completions.create(
94
extra_body={"guided_grammar": simplified_sql_grammar},
95
)
96
print(completion.choices[0].message.content)
97
+
98
+# Extra backend options
99
+prompt = ("Generate an email address for Alan Turing, who works in Enigma."
100
+ "End in .com and new line. Example result:"
101
+ "alan.turing@enigma.com\n")
102
+
103
+try:
104
+ # The no-fallback option forces vLLM to use xgrammar, so when it fails
105
+ # you get a 400 with the reason why
106
+ completion = client.chat.completions.create(
107
+ model="Qwen/Qwen2.5-3B-Instruct",
108
+ messages=[{
109
+ "role": "user",
110
+ "content": prompt,
111
+ }],
112
+ extra_body={
113
+ "guided_regex": "\w+@\w+\.com\n",
114
+ "stop": ["\n"],
115
+ "guided_decoding_backend": "xgrammar:no-fallback"
116
+ },
117
+ )
118
+except BadRequestError as e:
119
+ print("This error is expected:", e)
tests/entrypoints/llm/test_guided_generate.py
+16
@@ -280,6 +280,22 @@ def test_validation_against_both_guided_decoding_options(sample_regex, llm):
280
guided_options_request=dict(guided_regex=sample_regex))
281
282
283
+@pytest.mark.skip_global_cleanup
284
+def test_disable_guided_decoding_fallback(sample_regex, llm):
285
+ sampling_params = SamplingParams(temperature=0.8,
286
+ top_p=0.95,
287
+ guided_decoding=GuidedDecodingParams(
288
+ regex=sample_regex,
289
+ backend="xgrammar:no-fallback"))
290
+
291
+ with pytest.raises(
292
+ ValueError,
293
+ match="xgrammar does not support regex guided decoding"):
294
+ llm.generate(prompts="This should fail",
295
+ sampling_params=sampling_params,
296
+ use_tqdm=True)
297
+
298
+
299
@pytest.mark.skip_global_cleanup
300
@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)
301
def test_guided_json_object(llm, guided_decoding_backend: str):
tests/model_executor/test_guided_processors.py
+10
@@ -109,6 +109,16 @@ def test_multiple_guided_options_not_allowed(sample_json_schema, sample_regex):
109
GuidedDecodingParams(json=sample_json_schema, grammar="test grammar")
110
111
112
+def test_guided_decoding_backend_options():
113
+ """Test backend-specific options"""
114
+ params = GuidedDecodingParams(
115
+ backend="xgrammar:option-1,option-2,option-3")
116
+ assert params.backend_options() == ["option-1", "option-2", "option-3"]
117
+
118
+ no_fallback = GuidedDecodingParams(backend="xgrammar:option-1,no-fallback")
119
+ assert no_fallback.no_fallback()
120
+
121
+
122
def test_pickle_xgrammar_tokenizer_data():
123
124
# TODO: move to another test file for xgrammar
vllm/config.py
+4
-1
@@ -25,6 +25,7 @@ from vllm.model_executor.layers.quantization import (QUANTIZATION_METHODS,
25
get_quantization_config)
26
from vllm.model_executor.models import ModelRegistry
27
from vllm.platforms import CpuArchEnum
28
+from vllm.sampling_params import GuidedDecodingParams
29
from vllm.tracing import is_otel_available, otel_import_error_traceback
30
from vllm.transformers_utils.config import (
31
ConfigFormat, get_config, get_hf_image_processor_config,
@@ -2631,7 +2632,9 @@ class DecodingConfig:
2632
2633
def __post_init__(self):
2634
valid_guided_backends = ['outlines', 'lm-format-enforcer', 'xgrammar']
2634
- backend = self.guided_decoding_backend
2635
+
2636
+ backend = GuidedDecodingParams(
2637
+ backend=self.guided_decoding_backend).backend_name
2638
if backend not in valid_guided_backends:
2639
raise ValueError(f"Invalid guided_decoding_backend '{backend},"
2640
f"must be one of {valid_guided_backends}")
vllm/engine/arg_utils.py
+5
-2
@@ -372,14 +372,17 @@ class EngineArgs:
372
'--guided-decoding-backend',
373
type=str,
374
default='xgrammar',
375
- choices=['outlines', 'lm-format-enforcer', 'xgrammar'],
375
help='Which engine will be used for guided decoding'
376
' (JSON schema / regex etc) by default. Currently support '
377
'https://github.com/outlines-dev/outlines, '
378
'https://github.com/mlc-ai/xgrammar, and '
379
'https://github.com/noamgat/lm-format-enforcer.'
380
' Can be overridden per request via guided_decoding_backend'
382
- ' parameter.')
381
+ ' parameter.\n'
382
+ 'Backend-sepcific options can be supplied in a comma-separated '
383
+ 'list following a colon after the backend name. Valid backends and '
384
+ 'all available options are: [xgrammar:no-fallback, '
385
+ 'outlines:no-fallback, lm-format-enforcer:no-fallback]')
386
parser.add_argument(
387
'--logits-processor-pattern',
388
type=nullable_str,
vllm/model_executor/guided_decoding/__init__.py
+44
-37
@@ -22,47 +22,56 @@ logger = init_logger(__name__)
22
23
def maybe_backend_fallback(
24
guided_params: GuidedDecodingParams) -> GuidedDecodingParams:
25
+
26
+ def fallback_or_error(guided_params: GuidedDecodingParams, message: str,
27
+ fallback: str) -> None:
28
+ """Change the backend to the specified fallback with a warning log,
29
+ or raise a ValueError if the `no-fallback` option is specified."""
30
+ if guided_params.no_fallback():
31
+ raise ValueError(message)
32
+
33
+ logger.warning("%s Falling back to use %s instead.", message, fallback)
34
+ guided_params.backend = fallback
35
+
36
# lm-format-enforce doesn't support grammar, fallback to xgrammar
26
- if guided_params.backend == "lm-format-enforcer":
37
+ if guided_params.backend_name == "lm-format-enforcer":
38
if guided_params.grammar is not None:
28
- logger.warning(
29
- "lm-format-enforcer does not support grammar guided decoding. "
30
- "Falling back to use xgrammar instead.")
31
- guided_params.backend = "xgrammar"
39
+ fallback_or_error(
40
+ guided_params,
41
+ "lm-format-enforcer does not support grammar guided decoding.",
42
+ "xgrammar")
43
44
# lm-format-enforcer doesn't support some JSON schema features
45
elif (guided_params.json is not None
46
and has_lmf_unsupported_json_features(guided_params.json)):
36
- logger.warning(
47
+ fallback_or_error(
48
+ guided_params,
49
"lm-format-enforcer does not support advanced JSON schema "
38
- "features like patterns or numeric ranges. "
39
- "Falling back to use outlines instead.")
40
- guided_params.backend = "outlines"
50
+ "features like patterns or numeric ranges.", "outlines")
51
42
- if guided_params.backend == "xgrammar":
52
+ if guided_params.backend_name == "xgrammar":
53
from vllm.model_executor.guided_decoding.xgrammar_decoding import (
54
xgr_installed)
55
# xgrammar only has x86 wheels for linux, fallback to outlines
56
from vllm.platforms import current_platform
57
if current_platform.get_cpu_architecture() is not CpuArchEnum.X86:
48
- logger.warning("xgrammar is only supported on x86 CPUs. "
49
- "Falling back to use outlines instead.")
50
- guided_params.backend = "outlines"
58
+ fallback_or_error(guided_params,
59
+ "xgrammar is only supported on x86 CPUs.",
60
+ "outlines")
61
62
# xgrammar doesn't support regex, fallback to outlines
63
if guided_params.regex is not None:
54
- logger.warning("xgrammar does not support regex guided decoding. "
55
- "Falling back to use outlines instead.")
56
- guided_params.backend = "outlines"
64
+ fallback_or_error(
65
+ guided_params,
66
+ "xgrammar does not support regex guided decoding.", "outlines")
67
68
# xgrammar doesn't support some JSON schema features
69
elif (guided_params.json is not None
70
and has_xgrammar_unsupported_json_features(guided_params.json)):
61
- logger.warning(
71
+ fallback_or_error(
72
+ guided_params,
73
"xgrammar does not support advanced JSON schema features like "
63
- "patterns or numeric ranges. "
64
- "Falling back to use outlines instead.")
65
- guided_params.backend = "outlines"
74
+ "enums, patterns or numeric ranges.", "outlines")
75
76
# xgrammar only supports GBNF grammars, so we must convert Lark.
77
# We must check if the grammar is likely Lark and if that
@@ -72,25 +81,23 @@ def maybe_backend_fallback(
81
try:
82
convert_lark_to_gbnf(guided_params.grammar)
83
except Exception:
75
- logger.warning(
84
+ fallback_or_error(
85
+ guided_params,
86
"xgrammar does not support Lark grammars and the "
77
- "grammar failed to convert to GBNF. "
78
- "Falling back to use outlines instead.")
79
- guided_params.backend = "outlines"
87
+ "grammar failed to convert to GBNF.", "outlines")
88
89
# If the xgrammar module cannot be imported successfully,
90
# we should still allow users to use guided decoding with a fallback.
91
elif not xgr_installed:
84
- logger.warning("xgrammar module cannot be imported successfully. "
85
- "Falling back to use outlines instead.")
86
- guided_params.backend = "outlines"
92
+ fallback_or_error(
93
+ guided_params,
94
+ "xgrammar module cannot be imported successfully.", "outlines")
95
88
- if (guided_params.backend == "outlines"
96
+ if (guided_params.backend_name == "outlines"
97
and guided_params.json_object is not None):
98
# outlines doesn't support json_object, fallback to xgrammar
91
- logger.warning("outlines does not support json_object. "
92
- "Falling back to use xgrammar instead.")
93
- guided_params.backend = "xgrammar"
99
+ fallback_or_error(guided_params,
100
+ "outlines does not support json_object.", "xgrammar")
101
102
return guided_params
103
@@ -100,18 +107,18 @@ async def get_guided_decoding_logits_processor(
107
model_config: ModelConfig) -> LogitsProcessor | None:
108
guided_params = maybe_backend_fallback(guided_params)
109
# CFG grammar not supported by LMFE, so we use outlines instead
103
- if guided_params.backend == 'outlines':
110
+ if guided_params.backend_name == 'outlines':
111
# NOTE: lazy import outlines to avoid https://github.com/vllm-project/vllm/issues/4193
112
from vllm.model_executor.guided_decoding.outlines_decoding import ( # noqa
113
get_outlines_guided_decoding_logits_processor)
114
return await get_outlines_guided_decoding_logits_processor(
115
guided_params, tokenizer)
109
- if guided_params.backend == 'lm-format-enforcer':
116
+ if guided_params.backend_name == 'lm-format-enforcer':
117
from vllm.model_executor.guided_decoding.lm_format_enforcer_decoding import ( # noqa
118
get_local_lm_format_enforcer_guided_decoding_logits_processor)
119
return get_local_lm_format_enforcer_guided_decoding_logits_processor(
120
guided_params, tokenizer)
114
- if guided_params.backend == 'xgrammar':
121
+ if guided_params.backend_name == 'xgrammar':
122
from vllm.model_executor.guided_decoding.xgrammar_decoding import ( # noqa
123
get_local_xgrammar_guided_decoding_logits_processor)
124
return get_local_xgrammar_guided_decoding_logits_processor(
@@ -127,18 +134,18 @@ def get_local_guided_decoding_logits_processor(
134
model_config: ModelConfig) -> LogitsProcessor | None:
135
guided_params = maybe_backend_fallback(guided_params)
136
# CFG grammar not supported by LMFE, so we use outlines instead
130
- if guided_params.backend == 'outlines':
137
+ if guided_params.backend_name == 'outlines':
138
# NOTE: lazy import outlines to avoid https://github.com/vllm-project/vllm/issues/4193
139
from vllm.model_executor.guided_decoding.outlines_decoding import ( # noqa
140
get_local_outlines_guided_decoding_logits_processor)
141
return get_local_outlines_guided_decoding_logits_processor(
142
guided_params, tokenizer)
136
- if guided_params.backend == 'lm-format-enforcer':
143
+ if guided_params.backend_name == 'lm-format-enforcer':
144
from vllm.model_executor.guided_decoding.lm_format_enforcer_decoding import ( # noqa
145
get_local_lm_format_enforcer_guided_decoding_logits_processor)
146
return get_local_lm_format_enforcer_guided_decoding_logits_processor(
147
guided_params, tokenizer)
141
- if guided_params.backend == 'xgrammar':
148
+ if guided_params.backend_name == 'xgrammar':
149
from vllm.model_executor.guided_decoding.xgrammar_decoding import ( # noqa
150
get_local_xgrammar_guided_decoding_logits_processor)
151
return get_local_xgrammar_guided_decoding_logits_processor(
vllm/sampling_params.py
+19
@@ -64,6 +64,25 @@ class GuidedDecodingParams:
64
whitespace_pattern=whitespace_pattern,
65
)
66
67
+ @property
68
+ def backend_name(self) -> str:
69
+ """Return the backend name without any options.
70
+
71
+ For example if the backend is "xgrammar:no-fallback", returns "xgrammar"
72
+ """
73
+ return (self.backend or "").split(":")[0]
74
+
75
+ def backend_options(self) -> List[str]:
76
+ """Return the backend options as a list of strings."""
77
+ if not self.backend or ":" not in self.backend:
78
+ return []
79
+ return self.backend.split(":")[1].split(",")
80
+
81
+ def no_fallback(self) -> bool:
82
+ """Returns True if the "no-fallback" option is supplied for the guided
83
+ decoding backend"""
84
+ return "no-fallback" in self.backend_options()
85
+
86
def __post_init__(self):
87
"""Validate that some fields are mutually exclusive."""
88
guide_count = sum([