@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8d370e91cb
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[Bugfix] Fallback to outlines for complex json schemas (#10899)

Signed-off-by: mgoin <michael@neuralmagic.com>

Michael Goin committed Dec 4, 2024 at 22:14 UTC 8d370e91cb0049dc150c85710a08e85952504bfc
3 files changed +102
tests/entrypoints/conftest.py
+31
@@ -69,6 +69,37 @@ def sample_json_schema():
69 }
70
71
72 +@pytest.fixture
73 +def sample_complex_json_schema():
74 + return {
75 + "type": "object",
76 + "properties": {
77 + "score": {
78 + "type": "integer",
79 + "minimum": 0,
80 + "maximum": 100 # Numeric range
81 + },
82 + "grade": {
83 + "type": "string",
84 + "pattern": "^[A-D]$" # Regex pattern
85 + },
86 + "email": {
87 + "type": "string",
88 + "pattern": "^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}$"
89 + },
90 + "tags": {
91 + "type": "array",
92 + "items": {
93 + "type": "string",
94 + "pattern":
95 + "^[a-z]{1,10}$" # Combining length and pattern restrictions
96 + }
97 + }
98 + },
99 + "required": ["score", "grade", "email", "tags"]
100 + }
101 +
102 +
103 @pytest.fixture
104 def sample_guided_choice():
105 return [
tests/entrypoints/llm/test_guided_generate.py
+28
@@ -76,6 +76,34 @@ def test_guided_json_completion(sample_json_schema, llm):
76 jsonschema.validate(instance=output_json, schema=sample_json_schema)
77
78
79 +@pytest.mark.skip_global_cleanup
80 +def test_guided_complex_json_completion(sample_complex_json_schema, llm):
81 + sampling_params = SamplingParams(
82 + temperature=1.0,
83 + max_tokens=1000,
84 + guided_decoding=GuidedDecodingParams(json=sample_complex_json_schema))
85 + outputs = llm.generate(prompts=[
86 + f"Give an example JSON for an assignment grade "
87 + f"that fits this schema: {sample_complex_json_schema}"
88 + ] * 2,
89 + sampling_params=sampling_params,
90 + use_tqdm=True)
91 +
92 + assert outputs is not None
93 +
94 + for output in outputs:
95 + assert output is not None
96 + assert isinstance(output, RequestOutput)
97 + prompt = output.prompt
98 +
99 + generated_text = output.outputs[0].text
100 + assert generated_text is not None
101 + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
102 + output_json = json.loads(generated_text)
103 + jsonschema.validate(instance=output_json,
104 + schema=sample_complex_json_schema)
105 +
106 +
107 @pytest.mark.skip_global_cleanup
108 def test_guided_choice_completion(sample_guided_choice, llm):
109 sampling_params = SamplingParams(
vllm/model_executor/guided_decoding/__init__.py
+43
@@ -15,6 +15,40 @@ if TYPE_CHECKING:
15 logger = init_logger(__name__)
16
17
18 +def has_xgrammar_unsupported_json_features(schema: dict) -> bool:
19 + """Check if JSON schema contains features unsupported by xgrammar."""
20 +
21 + def check_object(obj: dict) -> bool:
22 + if not isinstance(obj, dict):
23 + return False
24 +
25 + # Check for pattern restrictions
26 + if "pattern" in obj:
27 + return True
28 +
29 + # Check for numeric ranges
30 + if obj.get("type") in ("integer", "number") and any(
31 + key in obj for key in [
32 + "minimum", "maximum", "exclusiveMinimum",
33 + "exclusiveMaximum", "multipleOf"
34 + ]):
35 + return True
36 +
37 + # Recursively check all nested objects and arrays
38 + for value in obj.values():
39 + if isinstance(value, dict):
40 + if check_object(value):
41 + return True
42 + elif isinstance(value, list):
43 + for item in value:
44 + if isinstance(item, dict) and check_object(item):
45 + return True
46 +
47 + return False
48 +
49 + return check_object(schema)
50 +
51 +
52 def maybe_backend_fallback(
53 guided_params: GuidedDecodingParams) -> GuidedDecodingParams:
54 # lm-format-enforce doesn't support grammar, fallback to xgrammar
@@ -47,6 +81,15 @@ def maybe_backend_fallback(
81 "Falling back to use outlines instead.")
82 guided_params.backend = "outlines"
83
84 + # xgrammar doesn't support some JSON schema features
85 + elif (guided_params.json is not None
86 + and has_xgrammar_unsupported_json_features(guided_params.json)):
87 + logger.warning(
88 + "xgrammar does not support advanced JSON schema features like "
89 + "patterns or numeric ranges. "
90 + "Falling back to use outlines instead.")
91 + guided_params.backend = "outlines"
92 +
93 return guided_params
94
95