A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 18:30 UTC
[V1] Add `disable-any-whitespace` option support for xgrammar (#15316)
Signed-off-by: Russell Bryant <rbryant@redhat.com>
Russell Bryant committed
Mar 22, 2025 at 11:56 UTC
eb63ea1e185846b4d02333b61f73a02fe60a242e
4 files changed
+53
-5
tests/v1/entrypoints/llm/test_struct_output_generate.py
+44
-1
@@ -57,6 +57,50 @@ def test_guided_json_completion(
57
jsonschema.validate(instance=output_json, schema=sample_json_schema)
58
59
60
+@pytest.mark.skip_global_cleanup
61
+@pytest.mark.parametrize("guided_decoding_backend",
62
+ GUIDED_DECODING_BACKENDS_V1)
63
+@pytest.mark.parametrize("model_name", MODELS_TO_TEST)
64
+def test_guided_json_completion_disable_any_whitespace(
65
+ monkeypatch: pytest.MonkeyPatch,
66
+ sample_json_schema: dict[str, Any],
67
+ guided_decoding_backend: str,
68
+ model_name: str,
69
+):
70
+ if guided_decoding_backend != "xgrammar":
71
+ pytest.skip("disable-any-whitespace is only supported for xgrammar.")
72
+ guided_decoding_backend = 'xgrammar:disable-any-whitespace'
73
+
74
+ monkeypatch.setenv("VLLM_USE_V1", "1")
75
+ llm = LLM(model=model_name,
76
+ max_model_len=1024,
77
+ guided_decoding_backend=guided_decoding_backend)
78
+ sampling_params = SamplingParams(
79
+ temperature=1.0,
80
+ max_tokens=1000,
81
+ guided_decoding=GuidedDecodingParams(json=sample_json_schema))
82
+ outputs = llm.generate(prompts=[
83
+ f"Give an example JSON for an employee profile "
84
+ f"that fits this schema: {sample_json_schema}"
85
+ ] * 2,
86
+ sampling_params=sampling_params,
87
+ use_tqdm=True)
88
+
89
+ assert outputs is not None
90
+
91
+ for output in outputs:
92
+ assert output is not None
93
+ assert isinstance(output, RequestOutput)
94
+ prompt = output.prompt
95
+
96
+ generated_text = output.outputs[0].text
97
+ assert generated_text is not None
98
+ assert "\n" not in generated_text
99
+ print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
100
+ output_json = json.loads(generated_text)
101
+ jsonschema.validate(instance=output_json, schema=sample_json_schema)
102
+
103
+
104
@pytest.mark.skip_global_cleanup
105
@pytest.mark.parametrize("guided_decoding_backend",
106
GUIDED_DECODING_BACKENDS_V1)
@@ -301,7 +345,6 @@ def test_guided_choice_completion(
345
prompts="The best language for type-safe systems programming is ",
346
sampling_params=sampling_params,
347
use_tqdm=True)
304
-
348
assert outputs is not None
349
for output in outputs:
350
assert output is not None
vllm/engine/arg_utils.py
+3
-1
@@ -1486,7 +1486,9 @@ class EngineArgs:
1486
return False
1487
1488
# Only support Xgrammar for guided decoding so far.
1489
- SUPPORTED_GUIDED_DECODING = ["xgrammar", "xgrammar:nofallback"]
1489
+ SUPPORTED_GUIDED_DECODING = [
1490
+ "xgrammar", "xgrammar:disable-any-whitespace"
1491
+ ]
1492
if self.guided_decoding_backend not in SUPPORTED_GUIDED_DECODING:
1493
_raise_or_fallback(feature_name="--guided-decoding-backend",
1494
recommend_to_remove=False)
vllm/v1/engine/processor.py
+1
-1
@@ -120,7 +120,7 @@ class Processor:
120
if not params.guided_decoding or not self.decoding_config:
121
return
122
123
- supported_backends = ["xgrammar"]
123
+ supported_backends = ["xgrammar", "xgrammar:disable-any-whitespace"]
124
engine_level_backend = self.decoding_config.guided_decoding_backend
125
if engine_level_backend not in supported_backends:
126
raise ValueError(f"Only {supported_backends} structured output is "
vllm/v1/structured_output/backend_xgrammar.py
+5
-2
@@ -26,6 +26,9 @@ class XgrammarBackend(StructuredOutputBackend):
26
27
def __init__(self, vllm_config: VllmConfig):
28
self.vllm_config = vllm_config
29
+ self.disable_any_whitespace = (
30
+ "disable-any-whitespace"
31
+ in vllm_config.decoding_config.guided_decoding_backend)
32
tokenizer_group = init_tokenizer_from_configs(
33
model_config=vllm_config.model_config,
34
scheduler_config=vllm_config.scheduler_config,
@@ -74,8 +77,8 @@ class XgrammarBackend(StructuredOutputBackend):
77
def compile_grammar(self, request_type: StructuredOutputOptions,
78
grammar_spec: str) -> StructuredOutputGrammar:
79
if request_type == StructuredOutputOptions.JSON:
77
- ctx = self.compiler.compile_json_schema(grammar_spec,
78
- any_whitespace=False)
80
+ ctx = self.compiler.compile_json_schema(
81
+ grammar_spec, any_whitespace=not self.disable_any_whitespace)
82
elif request_type == StructuredOutputOptions.JSON_OBJECT:
83
ctx = self.compiler.compile_builtin_json_grammar()
84
elif request_type == StructuredOutputOptions.GRAMMAR: