A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[Bugfix] Fallback to outlines for complex json schemas (#10899)
Signed-off-by: mgoin <michael@neuralmagic.com>
Michael Goin committed
Dec 4, 2024 at 22:14 UTC
8d370e91cb0049dc150c85710a08e85952504bfc
3 files changed
+102
tests/entrypoints/conftest.py
+31
@@ -69,6 +69,37 @@ def sample_json_schema():
69
}
70
71
72
+@pytest.fixture
73
+def sample_complex_json_schema():
74
+ return {
75
+ "type": "object",
76
+ "properties": {
77
+ "score": {
78
+ "type": "integer",
79
+ "minimum": 0,
80
+ "maximum": 100 # Numeric range
81
+ },
82
+ "grade": {
83
+ "type": "string",
84
+ "pattern": "^[A-D]$" # Regex pattern
85
+ },
86
+ "email": {
87
+ "type": "string",
88
+ "pattern": "^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}$"
89
+ },
90
+ "tags": {
91
+ "type": "array",
92
+ "items": {
93
+ "type": "string",
94
+ "pattern":
95
+ "^[a-z]{1,10}$" # Combining length and pattern restrictions
96
+ }
97
+ }
98
+ },
99
+ "required": ["score", "grade", "email", "tags"]
100
+ }
101
+
102
+
103
@pytest.fixture
104
def sample_guided_choice():
105
return [
tests/entrypoints/llm/test_guided_generate.py
+28
@@ -76,6 +76,34 @@ def test_guided_json_completion(sample_json_schema, llm):
76
jsonschema.validate(instance=output_json, schema=sample_json_schema)
77
78
79
+@pytest.mark.skip_global_cleanup
80
+def test_guided_complex_json_completion(sample_complex_json_schema, llm):
81
+ sampling_params = SamplingParams(
82
+ temperature=1.0,
83
+ max_tokens=1000,
84
+ guided_decoding=GuidedDecodingParams(json=sample_complex_json_schema))
85
+ outputs = llm.generate(prompts=[
86
+ f"Give an example JSON for an assignment grade "
87
+ f"that fits this schema: {sample_complex_json_schema}"
88
+ ] * 2,
89
+ sampling_params=sampling_params,
90
+ use_tqdm=True)
91
+
92
+ assert outputs is not None
93
+
94
+ for output in outputs:
95
+ assert output is not None
96
+ assert isinstance(output, RequestOutput)
97
+ prompt = output.prompt
98
+
99
+ generated_text = output.outputs[0].text
100
+ assert generated_text is not None
101
+ print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
102
+ output_json = json.loads(generated_text)
103
+ jsonschema.validate(instance=output_json,
104
+ schema=sample_complex_json_schema)
105
+
106
+
107
@pytest.mark.skip_global_cleanup
108
def test_guided_choice_completion(sample_guided_choice, llm):
109
sampling_params = SamplingParams(
vllm/model_executor/guided_decoding/__init__.py
+43
@@ -15,6 +15,40 @@ if TYPE_CHECKING:
15
logger = init_logger(__name__)
16
17
18
+def has_xgrammar_unsupported_json_features(schema: dict) -> bool:
19
+ """Check if JSON schema contains features unsupported by xgrammar."""
20
+
21
+ def check_object(obj: dict) -> bool:
22
+ if not isinstance(obj, dict):
23
+ return False
24
+
25
+ # Check for pattern restrictions
26
+ if "pattern" in obj:
27
+ return True
28
+
29
+ # Check for numeric ranges
30
+ if obj.get("type") in ("integer", "number") and any(
31
+ key in obj for key in [
32
+ "minimum", "maximum", "exclusiveMinimum",
33
+ "exclusiveMaximum", "multipleOf"
34
+ ]):
35
+ return True
36
+
37
+ # Recursively check all nested objects and arrays
38
+ for value in obj.values():
39
+ if isinstance(value, dict):
40
+ if check_object(value):
41
+ return True
42
+ elif isinstance(value, list):
43
+ for item in value:
44
+ if isinstance(item, dict) and check_object(item):
45
+ return True
46
+
47
+ return False
48
+
49
+ return check_object(schema)
50
+
51
+
52
def maybe_backend_fallback(
53
guided_params: GuidedDecodingParams) -> GuidedDecodingParams:
54
# lm-format-enforce doesn't support grammar, fallback to xgrammar
@@ -47,6 +81,15 @@ def maybe_backend_fallback(
81
"Falling back to use outlines instead.")
82
guided_params.backend = "outlines"
83
84
+ # xgrammar doesn't support some JSON schema features
85
+ elif (guided_params.json is not None
86
+ and has_xgrammar_unsupported_json_features(guided_params.json)):
87
+ logger.warning(
88
+ "xgrammar does not support advanced JSON schema features like "
89
+ "patterns or numeric ranges. "
90
+ "Falling back to use outlines instead.")
91
+ guided_params.backend = "outlines"
92
+
93
return guided_params
94
95