A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 16:00 UTC
[V1] Fix json_object support with xgrammar (#15488)
Signed-off-by: Russell Bryant <rbryant@redhat.com>
Russell Bryant committed
Apr 2, 2025 at 05:00 UTC
14e53ed11f5134381bde03484148cb3cd1b69cd6
5 files changed
+10
-19
requirements/common.txt
+1
-1
@@ -21,7 +21,7 @@ lm-format-enforcer >= 0.10.11, < 0.11
21
llguidance >= 0.7.9, < 0.8.0; platform_machine == "x86_64" or platform_machine == "arm64" or platform_machine == "aarch64"
22
outlines == 0.1.11
23
lark == 1.2.2
24
-xgrammar == 0.1.16; platform_machine == "x86_64" or platform_machine == "aarch64"
24
+xgrammar == 0.1.17; platform_machine == "x86_64" or platform_machine == "aarch64"
25
typing_extensions >= 4.10
26
filelock >= 3.16.1 # need to contain https://github.com/tox-dev/filelock/pull/317
27
partial-json-parser # used for parsing partial JSON outputs
tests/v1/entrypoints/llm/test_struct_output_generate.py
+2
-10
@@ -125,17 +125,9 @@ def test_structured_output(
125
print(generated_text)
126
assert generated_text is not None
127
128
- # Parse to verify it is valid JSON
128
+ # Parse to verify it is a valid JSON object
129
parsed_json = json.loads(generated_text)
130
- allowed_types: tuple[type, ...] = (dict, )
131
- if guided_decoding_backend.startswith("xgrammar"):
132
- # TODO - we are currently too permissive with xgrammar and
133
- # allow # any valid json (typically comes back as a list or
134
- # object). We can fix this by specifying a jsonschema of
135
- # {"type": "object"}, # but we need this fix in a release
136
- # first: https://github.com/mlc-ai/xgrammar/pull/264
137
- allowed_types = (dict, list)
138
- assert isinstance(parsed_json, allowed_types)
130
+ assert isinstance(parsed_json, dict)
131
132
#
133
# Test 3: test a jsonschema incompatible with xgrammar
vllm/model_executor/guided_decoding/__init__.py
-6
@@ -79,12 +79,6 @@ def maybe_backend_fallback(
79
"xgrammar does not support Lark grammars and the "
80
"grammar failed to convert to GBNF.", "outlines")
81
82
- elif guided_params.json_object:
83
- # https://github.com/mlc-ai/xgrammar/issues/256
84
- fallback_or_error(guided_params,
85
- "xgrammar does not support json_object.",
86
- "guidance")
87
-
82
# If the xgrammar module cannot be imported successfully,
83
# we should still allow users to use guided decoding with a fallback.
84
elif not xgr_installed:
vllm/model_executor/guided_decoding/xgrammar_decoding.py
+4
-1
@@ -320,7 +320,10 @@ class XGrammarLogitsProcessor:
320
elif self.config.grammar_str is not None:
321
self.ctx = compiler.compile_grammar(self.config.grammar_str)
322
elif self.config.json_object:
323
- self.ctx = compiler.compile_builtin_json_grammar()
323
+ any_whitespace = self.config.any_whitespace
324
+ self.ctx = compiler\
325
+ .compile_json_schema('{"type": "object"}',
326
+ any_whitespace=any_whitespace)
327
else:
328
raise ValueError(
329
"Invalid configuration for xgrammar logits processor")
vllm/v1/structured_output/backend_xgrammar.py
+3
-1
@@ -84,7 +84,9 @@ class XgrammarBackend(StructuredOutputBackend):
84
ctx = self.compiler.compile_json_schema(
85
grammar_spec, any_whitespace=not self.disable_any_whitespace)
86
elif request_type == StructuredOutputOptions.JSON_OBJECT:
87
- ctx = self.compiler.compile_builtin_json_grammar()
87
+ ctx = self.compiler.compile_json_schema(
88
+ '{"type": "object"}',
89
+ any_whitespace=not self.disable_any_whitespace)
90
elif request_type == StructuredOutputOptions.GRAMMAR:
91
ctx = self.compiler.compile_grammar(grammar_spec)
92
elif request_type == StructuredOutputOptions.REGEX: