A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 12:15 UTC
implement Structural Tag with Guidance backend (#17333)
Signed-off-by: Michal Moskal <michal@moskal.me>
Michał Moskal committed
Apr 28, 2025 at 19:21 UTC
86d9fc29cb39bbb09b5ac3202293eb6198666f18
2 files changed
+32
-10
tests/v1/entrypoints/llm/test_struct_output_generate.py
+4
-7
@@ -435,13 +435,10 @@ Given the previous instructions, what is the weather in New York City?
435
"""
436
437
# Change this once other backends support structural_tag
438
- if guided_decoding_backend.startswith("xgrammar"):
439
- outputs = llm.generate(prompts=prompt,
440
- sampling_params=sampling_params,
441
- use_tqdm=True)
442
- assert outputs is not None
443
- else:
444
- outputs = []
438
+ outputs = llm.generate(prompts=prompt,
439
+ sampling_params=sampling_params,
440
+ use_tqdm=True)
441
+ assert outputs is not None
442
443
for output in outputs:
444
assert output is not None
vllm/v1/structured_output/backend_guidance.py
+28
-3
@@ -173,7 +173,8 @@ def serialize_guidance_grammar(
173
disable_any_whitespace: bool = False,
174
no_additional_properties: bool = False,
175
) -> str:
176
- if request_type == StructuredOutputOptions.JSON:
176
+
177
+ def _process_schema(grammar_spec: Union[str, dict[str, Any]], ) -> str:
178
if no_additional_properties:
179
grammar_spec = process_for_additional_properties(grammar_spec)
180
return llguidance.LLMatcher.grammar_from_json_schema(
@@ -181,6 +182,9 @@ def serialize_guidance_grammar(
182
defaults={
183
"whitespace_flexible": not disable_any_whitespace,
184
})
185
+
186
+ if request_type == StructuredOutputOptions.JSON:
187
+ return _process_schema(grammar_spec)
188
elif request_type == StructuredOutputOptions.JSON_OBJECT:
189
return llguidance.LLMatcher.grammar_from_json_schema(
190
'{"type": "object"}',
@@ -195,8 +199,29 @@ def serialize_guidance_grammar(
199
elif request_type == StructuredOutputOptions.CHOICE:
200
tp = "choice"
201
elif request_type == StructuredOutputOptions.STRUCTURAL_TAG:
198
- raise ValueError("Structural tag is not supported "
199
- "for guidance backend yet")
202
+ if isinstance(grammar_spec, str):
203
+ s_tag = json.loads(grammar_spec)
204
+ else:
205
+ s_tag = grammar_spec
206
+ triggers: list[str] = s_tag["triggers"]
207
+ tags: list[llguidance.StructTag] = []
208
+ for s in s_tag["structures"]:
209
+ begin: str = s["begin"]
210
+ trig = next((t for t in triggers if begin.startswith(t)), None)
211
+ if trig is None:
212
+ raise ValueError(
213
+ f"Trigger {begin} not found in triggers {triggers}")
214
+ tags.append(
215
+ llguidance.StructTag(
216
+ trigger=trig,
217
+ begin=s["begin"],
218
+ grammar=_process_schema(s["schema"]),
219
+ end=s["end"],
220
+ ))
221
+ if not tags:
222
+ raise ValueError(
223
+ "No structural tags found in the grammar spec.")
224
+ return llguidance.StructTag.to_grammar(tags)
225
else:
226
logger.error("Validation should have already occurred. "
227
"Please file an issue.")