A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 17:15 UTC
[V1] Support disable_any_whtespace for guidance backend (#15584)
Signed-off-by: Russell Bryant <rbryant@redhat.com>
Russell Bryant committed
Mar 28, 2025 at 11:46 UTC
7329ff5468eceaf17f4b193ae3ef0b43c7bf38d6
6 files changed
+44
-117
tests/entrypoints/llm/test_guided_generate.py
+7
-55
@@ -6,7 +6,6 @@ import weakref
6
7
import jsonschema
8
import pytest
9
-from pydantic import BaseModel
9
10
from vllm.distributed import cleanup_dist_env_and_memory
11
from vllm.entrypoints.llm import LLM
@@ -15,7 +14,10 @@ from vllm.sampling_params import GuidedDecodingParams, SamplingParams
14
15
MODEL_NAME = "Qwen/Qwen2.5-1.5B-Instruct"
16
GUIDED_DECODING_BACKENDS = [
18
- "outlines", "lm-format-enforcer", "xgrammar", "guidance"
17
+ "outlines",
18
+ "lm-format-enforcer",
19
+ "xgrammar:disable-any-whitespace",
20
+ "guidance:disable-any-whitespace",
21
]
22
23
@@ -322,59 +324,9 @@ def test_guided_json_object(llm, guided_decoding_backend: str):
324
print(generated_text)
325
assert generated_text is not None
326
327
+ if 'disable-any-whitespace' in guided_decoding_backend:
328
+ assert "\n" not in generated_text
329
+
330
# Parse to verify it is valid JSON
331
parsed_json = json.loads(generated_text)
332
assert isinstance(parsed_json, dict)
328
-
329
-
330
-@pytest.mark.skip_global_cleanup
331
-def test_json_with_any_whitespace_disabled(llm):
332
-
333
- class ResponseSchema(BaseModel):
334
- clarifying_question: str
335
- cost_per_serving: str
336
- calories: str
337
- type_dish_ids: str
338
- type_meal_ids: str
339
- product_ids: list[str]
340
- exclude_product_ids: list[str]
341
- allergen_ids: list[str]
342
- total_cooking_time: str
343
- kitchen_ids: str
344
- holiday_ids: str
345
-
346
- # Note: Without this setting, the response is sometimes full of `\n`
347
- # for some models. This option prevents that.
348
- guided_decoding_backend = 'xgrammar:disable-any-whitespace'
349
-
350
- schema = ResponseSchema.model_json_schema()
351
- guided_params = GuidedDecodingParams(json=schema,
352
- backend=\
353
- guided_decoding_backend)
354
- sampling_params = SamplingParams(max_tokens=2000,
355
- frequency_penalty=0,
356
- presence_penalty=-1.1,
357
- repetition_penalty=1.3,
358
- guided_decoding=guided_params)
359
-
360
- prompt = ("<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You"
361
- "are a helpful assistant.<|im_end|>\n<|im_start|>user\nI want a "
362
- "quick launch fast with $10.<|im_end|>\n<|im_start|>assistant\n")
363
- outputs = llm.generate(prompts=prompt,
364
- sampling_params=sampling_params,
365
- use_tqdm=True)
366
-
367
- assert outputs is not None
368
-
369
- for output in outputs:
370
- assert output is not None
371
- assert isinstance(output, RequestOutput)
372
-
373
- generated_text = output.outputs[0].text
374
- assert generated_text is not None
375
- assert "\n" not in generated_text
376
-
377
- # Parse to verify it is valid JSON
378
- parsed_json = json.loads(generated_text)
379
- assert isinstance(parsed_json, dict)
380
- jsonschema.validate(instance=parsed_json, schema=schema)
tests/v1/entrypoints/llm/test_struct_output_generate.py
+7
-47
@@ -15,7 +15,9 @@ from vllm.entrypoints.llm import LLM
15
from vllm.outputs import RequestOutput
16
from vllm.sampling_params import GuidedDecodingParams, SamplingParams
17
18
-GUIDED_DECODING_BACKENDS_V1 = ["xgrammar", "guidance"]
18
+GUIDED_DECODING_BACKENDS_V1 = [
19
+ "xgrammar:disable-any-whitespace", "guidance:disable-any-whitespace"
20
+]
21
MODELS_TO_TEST = [
22
"Qwen/Qwen2.5-1.5B-Instruct", "mistralai/Ministral-8B-Instruct-2410"
23
]
@@ -55,50 +57,8 @@ def test_guided_json_completion(
57
58
generated_text = output.outputs[0].text
59
assert generated_text is not None
58
- print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
59
- output_json = json.loads(generated_text)
60
- jsonschema.validate(instance=output_json, schema=sample_json_schema)
61
-
62
-
63
-@pytest.mark.skip_global_cleanup
64
-@pytest.mark.parametrize("guided_decoding_backend",
65
- GUIDED_DECODING_BACKENDS_V1)
66
-@pytest.mark.parametrize("model_name", MODELS_TO_TEST)
67
-def test_guided_json_completion_disable_any_whitespace(
68
- monkeypatch: pytest.MonkeyPatch,
69
- sample_json_schema: dict[str, Any],
70
- guided_decoding_backend: str,
71
- model_name: str,
72
-):
73
- if guided_decoding_backend != "xgrammar":
74
- pytest.skip("disable-any-whitespace is only supported for xgrammar.")
75
- guided_decoding_backend = 'xgrammar:disable-any-whitespace'
76
-
77
- monkeypatch.setenv("VLLM_USE_V1", "1")
78
- llm = LLM(model=model_name,
79
- max_model_len=1024,
80
- guided_decoding_backend=guided_decoding_backend)
81
- sampling_params = SamplingParams(
82
- temperature=1.0,
83
- max_tokens=1000,
84
- guided_decoding=GuidedDecodingParams(json=sample_json_schema))
85
- outputs = llm.generate(prompts=[
86
- f"Give an example JSON for an employee profile "
87
- f"that fits this schema: {sample_json_schema}"
88
- ] * 2,
89
- sampling_params=sampling_params,
90
- use_tqdm=True)
91
-
92
- assert outputs is not None
93
-
94
- for output in outputs:
95
- assert output is not None
96
- assert isinstance(output, RequestOutput)
97
- prompt = output.prompt
98
-
99
- generated_text = output.outputs[0].text
100
- assert generated_text is not None
101
- assert "\n" not in generated_text
60
+ if 'disable-any-whitespace' in guided_decoding_backend:
61
+ assert "\n" not in generated_text
62
print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
63
output_json = json.loads(generated_text)
64
jsonschema.validate(instance=output_json, schema=sample_json_schema)
@@ -142,7 +102,7 @@ def test_guided_json_object(
102
# Parse to verify it is valid JSON
103
parsed_json = json.loads(generated_text)
104
allowed_types: tuple[type, ...] = (dict, )
145
- if guided_decoding_backend == "xgrammar":
105
+ if guided_decoding_backend.startswith("xgrammar"):
106
# TODO - we are currently too permissive with xgrammar and
107
# allow # any valid json (typically comes back as a list or
108
# object). We can fix this by specifying a jsonschema of
@@ -170,7 +130,7 @@ def test_guided_json_unsupported_schema(
130
temperature=1.0,
131
max_tokens=1000,
132
guided_decoding=GuidedDecodingParams(json=unsupported_json_schema))
173
- if guided_decoding_backend == "xgrammar":
133
+ if guided_decoding_backend.startswith("xgrammar"):
134
with pytest.raises(ValueError,
135
match="The provided JSON schema contains features "
136
"not supported by xgrammar."):
vllm/engine/arg_utils.py
+2
-1
@@ -1561,7 +1561,8 @@ class EngineArgs:
1561
1562
# Xgrammar and Guidance are supported.
1563
SUPPORTED_GUIDED_DECODING = [
1564
- "xgrammar", "xgrammar:disable-any-whitespace", "guidance", "auto"
1564
+ "xgrammar", "xgrammar:disable-any-whitespace", "guidance",
1565
+ "guidance:disable-any-whitespace", "auto"
1566
]
1567
if self.guided_decoding_backend not in SUPPORTED_GUIDED_DECODING:
1568
_raise_or_fallback(feature_name="--guided-decoding-backend",
vllm/model_executor/guided_decoding/guidance_decoding.py
+10
-2
@@ -18,14 +18,22 @@ def get_local_guidance_guided_decoding_logits_processor(
18
"""
19
20
grm = ""
21
+ any_whitespace = 'disable-any-whitespace' not in \
22
+ guided_params.backend_options()
23
if guided_params.json:
24
grm = llguidance.LLMatcher.grammar_from_json_schema(
25
guided_params.json,
24
- overrides={"whitespace_pattern": guided_params.whitespace_pattern})
26
+ overrides={"whitespace_pattern": guided_params.whitespace_pattern},
27
+ defaults={
28
+ "whitespace_flexible": any_whitespace,
29
+ })
30
elif guided_params.json_object:
31
grm = llguidance.LLMatcher.grammar_from_json_schema(
32
'{"type": "object"}',
28
- overrides={"whitespace_pattern": guided_params.whitespace_pattern})
33
+ overrides={"whitespace_pattern": guided_params.whitespace_pattern},
34
+ defaults={
35
+ "whitespace_flexible": any_whitespace,
36
+ })
37
elif guided_params.regex:
38
grm = llguidance.grammar_from("regex", guided_params.regex)
39
elif guided_params.choice:
vllm/v1/engine/processor.py
+6
-5
@@ -121,7 +121,8 @@ class Processor:
121
return
122
123
supported_backends = [
124
- "xgrammar", "xgrammar:disable-any-whitespace", "guidance", "auto"
124
+ "xgrammar", "xgrammar:disable-any-whitespace", "guidance",
125
+ "guidance:disable-any-whitespace", "auto"
126
]
127
engine_level_backend = self.decoding_config.guided_decoding_backend
128
if engine_level_backend not in supported_backends:
@@ -140,11 +141,10 @@ class Processor:
141
raise ValueError("Structured output is not supported on TPU.")
142
143
# Request content validation
143
-
144
- if engine_level_backend == "xgrammar":
144
+ if engine_level_backend.startswith("xgrammar"):
145
# xgrammar with no fallback
146
validate_structured_output_request_xgrammar(params)
147
- params.guided_decoding.backend = "xgrammar"
147
+ params.guided_decoding.backend = engine_level_backend
148
elif engine_level_backend == "auto":
149
# "auto" is an opt-in to opinionated behavior where we try to
150
# choose a backend based on request contents. This is not the
@@ -158,12 +158,13 @@ class Processor:
158
# are not supported in xgrammar. Fall back to guidance.
159
params.guided_decoding.backend = "guidance"
160
161
- if params.guided_decoding.backend == "guidance":
161
+ if engine_level_backend.startswith("guidance"):
162
# TODO ideally we would have the LLTokenizer here as Lark syntax
163
# allows <|special_token|> and similar, see
164
# https://github.com/guidance-ai/llguidance/blob/main/docs/syntax.md#special-tokens
165
# Without tokenizer these are disallowed in grammars.
166
validate_guidance_grammar(params, tokenizer=None)
167
+ params.guided_decoding.backend = engine_level_backend
168
169
def process_inputs(
170
self,
vllm/v1/structured_output/backend_guidance.py
+12
-7
@@ -41,6 +41,9 @@ class GuidanceBackend(StructuredOutputBackend):
41
tokenizer_group.ping()
42
self.vllm_config = vllm_config
43
self.vocab_size = vllm_config.model_config.get_vocab_size()
44
+ self.disable_any_whitespace = (
45
+ "disable-any-whitespace"
46
+ in vllm_config.decoding_config.guided_decoding_backend)
47
48
tokenizer = tokenizer_group.get_lora_tokenizer(None)
49
self.ll_tokenizer = llguidance_hf.from_tokenizer(tokenizer, None)
@@ -48,7 +51,7 @@ class GuidanceBackend(StructuredOutputBackend):
51
def compile_grammar(self, request_type: StructuredOutputOptions,
52
grammar_spec: str) -> StructuredOutputGrammar:
53
self.serialized_grammar = serialize_guidance_grammar(
51
- request_type, grammar_spec)
54
+ request_type, grammar_spec, self.disable_any_whitespace)
55
56
ll_matcher = llguidance.LLMatcher(
57
self.ll_tokenizer,
@@ -126,17 +129,19 @@ class GuidanceGrammar(StructuredOutputGrammar):
129
130
131
def serialize_guidance_grammar(request_type: StructuredOutputOptions,
129
- grammar_spec: str) -> str:
132
+ grammar_spec: str,
133
+ disable_any_whitespace: bool = False) -> str:
134
if request_type == StructuredOutputOptions.JSON:
131
- # TODO: make whitespace_flexible configurable
135
return llguidance.LLMatcher.grammar_from_json_schema(
133
- grammar_spec, defaults={
134
- "whitespace_flexible": True,
136
+ grammar_spec,
137
+ defaults={
138
+ "whitespace_flexible": not disable_any_whitespace,
139
})
140
elif request_type == StructuredOutputOptions.JSON_OBJECT:
141
return llguidance.LLMatcher.grammar_from_json_schema(
138
- '{"type": "object"}', defaults={
139
- "whitespace_flexible": True,
142
+ '{"type": "object"}',
143
+ defaults={
144
+ "whitespace_flexible": not disable_any_whitespace,
145
})
146
else:
147
if request_type == StructuredOutputOptions.REGEX: