A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 14:45 UTC
[V1] Set structured output backend to `auto` by default (#15724)
Signed-off-by: Russell Bryant <rbryant@redhat.com>
Russell Bryant committed
Apr 10, 2025 at 13:53 UTC
9665313c3986e0ff7d9047d62cf47ddaa1dba485
4 files changed
+22
-68
tests/entrypoints/openai/test_chat.py
+11
-63
@@ -20,8 +20,6 @@ from .test_completion import zephyr_lora_files # noqa: F401
20
# any model with a chat template should work here
21
MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
22
23
-GUIDED_DECODING_BACKENDS = ["outlines", "lm-format-enforcer", "xgrammar"]
24
-
23
24
@pytest.fixture(scope="module")
25
def monkeypatch_module():
@@ -487,20 +485,9 @@ async def test_chat_completion_stream_options(client: openai.AsyncOpenAI,
485
assert last_completion_tokens == 10
486
487
490
-# NOTE: Not sure why, but when I place this after `test_guided_regex_chat`
491
-# (i.e. using the same ordering as in the Completions API tests), the test
492
-# will fail on the second `guided_decoding_backend` even when I swap their order
493
-# (ref: https://github.com/vllm-project/vllm/pull/5526#issuecomment-2173772256)
488
@pytest.mark.asyncio
495
-@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)
489
async def test_guided_choice_chat(client: openai.AsyncOpenAI,
497
- is_v1_server: bool,
498
- guided_decoding_backend: str,
490
sample_guided_choice):
500
-
501
- if is_v1_server and guided_decoding_backend != 'xgrammar':
502
- pytest.skip("Only xgrammar backend is supported with V1")
503
-
491
messages = [{
492
"role": "system",
493
"content": "you are a helpful assistant"
@@ -515,8 +502,7 @@ async def test_guided_choice_chat(client: openai.AsyncOpenAI,
502
messages=messages,
503
max_completion_tokens=10,
504
temperature=0.7,
518
- extra_body=dict(guided_choice=sample_guided_choice,
519
- guided_decoding_backend=guided_decoding_backend))
505
+ extra_body=dict(guided_choice=sample_guided_choice))
506
choice1 = chat_completion.choices[0].message.content
507
assert choice1 in sample_guided_choice
508
@@ -530,22 +516,16 @@ async def test_guided_choice_chat(client: openai.AsyncOpenAI,
516
messages=messages,
517
max_completion_tokens=10,
518
temperature=0.7,
533
- extra_body=dict(guided_choice=sample_guided_choice,
534
- guided_decoding_backend=guided_decoding_backend))
519
+ extra_body=dict(guided_choice=sample_guided_choice))
520
choice2 = chat_completion.choices[0].message.content
521
assert choice2 in sample_guided_choice
522
assert choice1 != choice2
523
524
525
@pytest.mark.asyncio
541
-@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)
542
-async def test_guided_json_chat(client: openai.AsyncOpenAI, is_v1_server: bool,
543
- guided_decoding_backend: str,
526
+async def test_guided_json_chat(client: openai.AsyncOpenAI,
527
sample_json_schema):
528
546
- if is_v1_server:
547
- pytest.skip("sample_json_schema has features unsupported in V1")
548
-
529
messages = [{
530
"role": "system",
531
"content": "you are a helpful assistant"
@@ -560,8 +540,7 @@ async def test_guided_json_chat(client: openai.AsyncOpenAI, is_v1_server: bool,
540
model=MODEL_NAME,
541
messages=messages,
542
max_completion_tokens=1000,
563
- extra_body=dict(guided_json=sample_json_schema,
564
- guided_decoding_backend=guided_decoding_backend))
543
+ extra_body=dict(guided_json=sample_json_schema))
544
message = chat_completion.choices[0].message
545
assert message.content is not None
546
json1 = json.loads(message.content)
@@ -578,8 +557,7 @@ async def test_guided_json_chat(client: openai.AsyncOpenAI, is_v1_server: bool,
557
model=MODEL_NAME,
558
messages=messages,
559
max_completion_tokens=1000,
581
- extra_body=dict(guided_json=sample_json_schema,
582
- guided_decoding_backend=guided_decoding_backend))
560
+ extra_body=dict(guided_json=sample_json_schema))
561
message = chat_completion.choices[0].message
562
assert message.content is not None
563
json2 = json.loads(message.content)
@@ -589,13 +567,7 @@ async def test_guided_json_chat(client: openai.AsyncOpenAI, is_v1_server: bool,
567
568
569
@pytest.mark.asyncio
592
-@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)
593
-async def test_guided_regex_chat(client: openai.AsyncOpenAI,
594
- is_v1_server: bool,
595
- guided_decoding_backend: str, sample_regex):
596
-
597
- if is_v1_server and guided_decoding_backend != 'xgrammar':
598
- pytest.skip("Only xgrammar backend is supported with V1")
570
+async def test_guided_regex_chat(client: openai.AsyncOpenAI, sample_regex):
571
572
messages = [{
573
"role": "system",
@@ -610,8 +582,7 @@ async def test_guided_regex_chat(client: openai.AsyncOpenAI,
582
model=MODEL_NAME,
583
messages=messages,
584
max_completion_tokens=20,
613
- extra_body=dict(guided_regex=sample_regex,
614
- guided_decoding_backend=guided_decoding_backend))
585
+ extra_body=dict(guided_regex=sample_regex))
586
ip1 = chat_completion.choices[0].message.content
587
assert ip1 is not None
588
assert re.fullmatch(sample_regex, ip1) is not None
@@ -622,8 +593,7 @@ async def test_guided_regex_chat(client: openai.AsyncOpenAI,
593
model=MODEL_NAME,
594
messages=messages,
595
max_completion_tokens=20,
625
- extra_body=dict(guided_regex=sample_regex,
626
- guided_decoding_backend=guided_decoding_backend))
596
+ extra_body=dict(guided_regex=sample_regex))
597
ip2 = chat_completion.choices[0].message.content
598
assert ip2 is not None
599
assert re.fullmatch(sample_regex, ip2) is not None
@@ -652,15 +622,9 @@ async def test_guided_decoding_type_error(client: openai.AsyncOpenAI):
622
623
624
@pytest.mark.asyncio
655
-@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)
625
async def test_guided_choice_chat_logprobs(client: openai.AsyncOpenAI,
657
- is_v1_server: bool,
658
- guided_decoding_backend: str,
626
sample_guided_choice):
627
661
- if is_v1_server and guided_decoding_backend != 'xgrammar':
662
- pytest.skip("Only xgrammar backend is supported with V1")
663
-
628
messages = [{
629
"role": "system",
630
"content": "you are a helpful assistant"
@@ -676,8 +640,7 @@ async def test_guided_choice_chat_logprobs(client: openai.AsyncOpenAI,
640
max_completion_tokens=10,
641
logprobs=True,
642
top_logprobs=5,
679
- extra_body=dict(guided_choice=sample_guided_choice,
680
- guided_decoding_backend=guided_decoding_backend))
643
+ extra_body=dict(guided_choice=sample_guided_choice))
644
645
assert chat_completion.choices[0].logprobs is not None
646
assert chat_completion.choices[0].logprobs.content is not None
@@ -689,14 +652,7 @@ async def test_guided_choice_chat_logprobs(client: openai.AsyncOpenAI,
652
653
654
@pytest.mark.asyncio
692
-@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)
693
-async def test_named_tool_use(client: openai.AsyncOpenAI, is_v1_server: bool,
694
- guided_decoding_backend: str,
695
- sample_json_schema):
696
-
697
- if is_v1_server:
698
- pytest.skip("sample_json_schema has features unsupported on V1")
699
-
655
+async def test_named_tool_use(client: openai.AsyncOpenAI, sample_json_schema):
656
messages = [{
657
"role": "system",
658
"content": "you are a helpful assistant"
@@ -728,7 +684,7 @@ async def test_named_tool_use(client: openai.AsyncOpenAI, is_v1_server: bool,
684
"name": "dummy_function_name"
685
}
686
},
731
- extra_body=dict(guided_decoding_backend=guided_decoding_backend))
687
+ )
688
message = chat_completion.choices[0].message
689
assert len(message.content) == 0
690
json_string = message.tool_calls[0].function.arguments
@@ -763,7 +719,6 @@ async def test_named_tool_use(client: openai.AsyncOpenAI, is_v1_server: bool,
719
"name": "dummy_function_name"
720
}
721
},
766
- extra_body=dict(guided_decoding_backend=guided_decoding_backend),
722
stream=True)
723
724
output = []
@@ -888,7 +843,6 @@ async def test_required_tool_use(client: openai.AsyncOpenAI,
843
model=model_name,
844
tools=tools,
845
tool_choice="required",
891
- extra_body=dict(guided_decoding_backend="outlines"),
846
)
847
848
assert chat_completion.choices[0].message.tool_calls is not None
@@ -900,7 +854,6 @@ async def test_required_tool_use(client: openai.AsyncOpenAI,
854
model=model_name,
855
tools=tools,
856
tool_choice="required",
903
- extra_body=dict(guided_decoding_backend="outlines"),
857
stream=True,
858
)
859
@@ -914,12 +867,7 @@ async def test_required_tool_use(client: openai.AsyncOpenAI,
867
868
@pytest.mark.asyncio
869
async def test_inconsistent_tool_choice_and_tools(client: openai.AsyncOpenAI,
917
- is_v1_server: bool,
870
sample_json_schema):
919
-
920
- if is_v1_server:
921
- pytest.skip("sample_json_schema has features unsupported on V1")
922
-
871
messages = [{
872
"role": "system",
873
"content": "you are a helpful assistant"
vllm/config.py
+2
-2
@@ -2976,7 +2976,7 @@ class DecodingConfig:
2976
2977
# Which guided decoding algo to use.
2978
# 'outlines' / 'lm-format-enforcer' / 'xgrammar'
2979
- guided_decoding_backend: str = 'xgrammar'
2979
+ guided_decoding_backend: str = "auto" if envs.VLLM_USE_V1 else "xgrammar"
2980
2981
reasoning_backend: Optional[str] = None
2982
@@ -3001,7 +3001,7 @@ class DecodingConfig:
3001
3002
def __post_init__(self):
3003
v0_valid_guided_backends = [
3004
- 'outlines', 'lm-format-enforcer', 'xgrammar'
3004
+ 'outlines', 'lm-format-enforcer', 'xgrammar', 'auto'
3005
]
3006
v1_valid_guided_backends = ['xgrammar', 'guidance', 'auto']
3007
vllm/engine/arg_utils.py
+3
-3
@@ -182,7 +182,7 @@ class EngineArgs:
182
enable_chunked_prefill: Optional[bool] = None
183
disable_chunked_mm_input: bool = False
184
185
- guided_decoding_backend: str = 'xgrammar'
185
+ guided_decoding_backend: str = DecodingConfig.guided_decoding_backend
186
logits_processor_pattern: Optional[str] = None
187
188
speculative_config: Optional[Dict[str, Any]] = None
@@ -407,13 +407,13 @@ class EngineArgs:
407
parser.add_argument(
408
'--guided-decoding-backend',
409
type=str,
410
- default='xgrammar',
410
+ default=DecodingConfig.guided_decoding_backend,
411
help='Which engine will be used for guided decoding'
412
' (JSON schema / regex etc) by default. Currently support '
413
'https://github.com/mlc-ai/xgrammar and '
414
'https://github.com/guidance-ai/llguidance.'
415
'Valid backend values are "xgrammar", "guidance", and "auto". '
416
- 'With "auto", we will make opinionated choices based on request'
416
+ 'With "auto", we will make opinionated choices based on request '
417
'contents and what the backend libraries currently support, so '
418
'the behavior is subject to change in each release.')
419
parser.add_argument(
vllm/model_executor/guided_decoding/__init__.py
+6
@@ -33,6 +33,12 @@ def maybe_backend_fallback(
33
logger.warning("%s Falling back to use %s instead.", message, fallback)
34
guided_params.backend = fallback
35
36
+ # `auto` was added for V1 to explicitly declare a mode that has fallbacks
37
+ # in place. If that is specified with V0, treat it as `xgrammar`, as we have
38
+ # fallbacks enabled for that and it is the V0 default.
39
+ if guided_params.backend == "auto":
40
+ guided_params.backend = "xgrammar"
41
+
42
# lm-format-enforce doesn't support grammar, fallback to xgrammar
43
if guided_params.backend_name == "lm-format-enforcer":
44
if guided_params.grammar is not None: