A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[CI] Expand test_guided_generate to test all backends (#11313)
Signed-off-by: mgoin <michael@neuralmagic.com>
Michael Goin committed
Dec 18, 2024 at 23:00 UTC
a30482f0545a216d6c02241f6663b9cddbcb066e
3 files changed
+129
-51
tests/entrypoints/llm/test_guided_generate.py
+69
-43
@@ -10,7 +10,8 @@ from vllm.entrypoints.llm import LLM
10
from vllm.outputs import RequestOutput
11
from vllm.sampling_params import GuidedDecodingParams, SamplingParams
12
13
-MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
13
+MODEL_NAME = "Qwen/Qwen2.5-7B-Instruct"
14
+GUIDED_DECODING_BACKENDS = ["outlines", "lm-format-enforcer", "xgrammar"]
15
16
17
@pytest.fixture(scope="module")
@@ -26,11 +27,13 @@ def llm():
27
28
29
@pytest.mark.skip_global_cleanup
29
-def test_guided_regex(sample_regex, llm):
30
- sampling_params = SamplingParams(
31
- temperature=0.8,
32
- top_p=0.95,
33
- guided_decoding=GuidedDecodingParams(regex=sample_regex))
30
+@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)
31
+def test_guided_regex(sample_regex, llm, guided_decoding_backend: str):
32
+ sampling_params = SamplingParams(temperature=0.8,
33
+ top_p=0.95,
34
+ guided_decoding=GuidedDecodingParams(
35
+ regex=sample_regex,
36
+ backend=guided_decoding_backend))
37
outputs = llm.generate(prompts=[
38
f"Give an example IPv4 address with this regex: {sample_regex}"
39
] * 2,
@@ -50,11 +53,14 @@ def test_guided_regex(sample_regex, llm):
53
54
55
@pytest.mark.skip_global_cleanup
53
-def test_guided_json_completion(sample_json_schema, llm):
54
- sampling_params = SamplingParams(
55
- temperature=1.0,
56
- max_tokens=1000,
57
- guided_decoding=GuidedDecodingParams(json=sample_json_schema))
56
+@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)
57
+def test_guided_json_completion(sample_json_schema, llm,
58
+ guided_decoding_backend: str):
59
+ sampling_params = SamplingParams(temperature=1.0,
60
+ max_tokens=1000,
61
+ guided_decoding=GuidedDecodingParams(
62
+ json=sample_json_schema,
63
+ backend=guided_decoding_backend))
64
outputs = llm.generate(prompts=[
65
f"Give an example JSON for an employee profile "
66
f"that fits this schema: {sample_json_schema}"
@@ -77,11 +83,14 @@ def test_guided_json_completion(sample_json_schema, llm):
83
84
85
@pytest.mark.skip_global_cleanup
80
-def test_guided_complex_json_completion(sample_complex_json_schema, llm):
81
- sampling_params = SamplingParams(
82
- temperature=1.0,
83
- max_tokens=1000,
84
- guided_decoding=GuidedDecodingParams(json=sample_complex_json_schema))
86
+@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)
87
+def test_guided_complex_json_completion(sample_complex_json_schema, llm,
88
+ guided_decoding_backend: str):
89
+ sampling_params = SamplingParams(temperature=1.0,
90
+ max_tokens=1000,
91
+ guided_decoding=GuidedDecodingParams(
92
+ json=sample_complex_json_schema,
93
+ backend=guided_decoding_backend))
94
outputs = llm.generate(prompts=[
95
f"Give an example JSON for an assignment grade "
96
f"that fits this schema: {sample_complex_json_schema}"
@@ -105,11 +114,14 @@ def test_guided_complex_json_completion(sample_complex_json_schema, llm):
114
115
116
@pytest.mark.skip_global_cleanup
108
-def test_guided_definition_json_completion(sample_definition_json_schema, llm):
117
+@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)
118
+def test_guided_definition_json_completion(sample_definition_json_schema, llm,
119
+ guided_decoding_backend: str):
120
sampling_params = SamplingParams(temperature=1.0,
121
max_tokens=1000,
122
guided_decoding=GuidedDecodingParams(
112
- json=sample_definition_json_schema))
123
+ json=sample_definition_json_schema,
124
+ backend=guided_decoding_backend))
125
outputs = llm.generate(prompts=[
126
f"Give an example JSON for solving 8x + 7 = -23 "
127
f"that fits this schema: {sample_definition_json_schema}"
@@ -133,11 +145,14 @@ def test_guided_definition_json_completion(sample_definition_json_schema, llm):
145
146
147
@pytest.mark.skip_global_cleanup
136
-def test_guided_choice_completion(sample_guided_choice, llm):
137
- sampling_params = SamplingParams(
138
- temperature=0.8,
139
- top_p=0.95,
140
- guided_decoding=GuidedDecodingParams(choice=sample_guided_choice))
148
+@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)
149
+def test_guided_choice_completion(sample_guided_choice, llm,
150
+ guided_decoding_backend: str):
151
+ sampling_params = SamplingParams(temperature=0.8,
152
+ top_p=0.95,
153
+ guided_decoding=GuidedDecodingParams(
154
+ choice=sample_guided_choice,
155
+ backend=guided_decoding_backend))
156
outputs = llm.generate(
157
prompts="The best language for type-safe systems programming is ",
158
sampling_params=sampling_params,
@@ -156,13 +171,20 @@ def test_guided_choice_completion(sample_guided_choice, llm):
171
172
173
@pytest.mark.skip_global_cleanup
159
-def test_guided_grammar(sample_sql_statements, llm):
160
-
161
- sampling_params = SamplingParams(
162
- temperature=0.8,
163
- top_p=0.95,
164
- max_tokens=1000,
165
- guided_decoding=GuidedDecodingParams(grammar=sample_sql_statements))
174
+@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)
175
+def test_guided_grammar(sample_sql_statements, llm,
176
+ guided_decoding_backend: str):
177
+ if guided_decoding_backend == "outlines":
178
+ pytest.skip("Outlines backend fails in this test case with:\n"
179
+ "AttributeError: Error in model execution: 'ParserConf' "
180
+ "object has no attribute 'deterministic'")
181
+
182
+ sampling_params = SamplingParams(temperature=0.8,
183
+ top_p=0.95,
184
+ max_tokens=1000,
185
+ guided_decoding=GuidedDecodingParams(
186
+ grammar=sample_sql_statements,
187
+ backend=guided_decoding_backend))
188
outputs = llm.generate(
189
prompts=("Generate a sql state that select col_1 from "
190
"table_1 where it is equals to 1"),
@@ -218,15 +240,18 @@ def test_validation_against_both_guided_decoding_options(sample_regex, llm):
240
241
242
@pytest.mark.skip_global_cleanup
221
-def test_guided_json_object(llm):
222
- sampling_params = SamplingParams(
223
- temperature=1.0,
224
- max_tokens=100,
225
- guided_decoding=GuidedDecodingParams(json_object=True))
243
+@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)
244
+def test_guided_json_object(llm, guided_decoding_backend: str):
245
+ sampling_params = SamplingParams(temperature=1.0,
246
+ max_tokens=100,
247
+ n=2,
248
+ guided_decoding=GuidedDecodingParams(
249
+ json_object=True,
250
+ backend=guided_decoding_backend))
251
252
outputs = llm.generate(
228
- prompts=("Generate a JSON object describing a person with name "
229
- "and age for John Smith who is 31 years old."),
253
+ prompts=("Generate a JSON object with curly braces for a person with "
254
+ "name and age fields for John Smith who is 31 years old."),
255
sampling_params=sampling_params,
256
use_tqdm=True)
257
@@ -235,10 +260,11 @@ def test_guided_json_object(llm):
260
assert output is not None
261
assert isinstance(output, RequestOutput)
262
238
- generated_text = output.outputs[0].text
239
- print(generated_text)
240
- assert generated_text is not None
263
+ for i in range(2):
264
+ generated_text = output.outputs[i].text
265
+ print(generated_text)
266
+ assert generated_text is not None
267
242
- # Parse to verify it is valid JSON
243
- parsed_json = json.loads(generated_text)
244
- assert isinstance(parsed_json, dict)
268
+ # Parse to verify it is valid JSON
269
+ parsed_json = json.loads(generated_text)
270
+ assert isinstance(parsed_json, dict)
tests/model_executor/test_guided_processors.py
+2
-2
@@ -13,6 +13,7 @@ from vllm.model_executor.guided_decoding.outlines_logits_processors import (
13
from vllm.sampling_params import GuidedDecodingParams
14
15
MODEL_NAME = 'HuggingFaceH4/zephyr-7b-beta'
16
+GUIDED_DECODING_BACKENDS = ["outlines", "lm-format-enforcer", "xgrammar"]
17
18
19
def test_guided_logits_processors(sample_regex, sample_json_schema):
@@ -42,8 +43,7 @@ def test_guided_logits_processors(sample_regex, sample_json_schema):
43
44
45
@pytest.mark.asyncio
45
-@pytest.mark.parametrize("backend",
46
- ["outlines", "lm-format-enforcer", "xgrammar"])
46
+@pytest.mark.parametrize("backend", GUIDED_DECODING_BACKENDS)
47
@pytest.mark.parametrize("is_local", [True, False])
48
async def test_guided_logits_processor_black_box(backend: str, is_local: bool,
49
sample_regex,
vllm/model_executor/guided_decoding/__init__.py
+58
-6
@@ -49,15 +49,60 @@ def has_xgrammar_unsupported_json_features(schema: dict) -> bool:
49
return check_object(schema)
50
51
52
+def has_lmf_unsupported_json_features(schema: dict) -> bool:
53
+ """
54
+ Check if JSON schema contains features unsupported
55
+ by lm_format_enforcer.
56
+
57
+ Known issues:
58
+ - Regex patterns:
59
+ "grade": {
60
+ "type": "string",
61
+ "pattern": "^[A-D]$" # Regex pattern
62
+ },
63
+ """
64
+
65
+ def check_object(obj: dict) -> bool:
66
+ if not isinstance(obj, dict):
67
+ return False
68
+
69
+ # Check for pattern restrictions
70
+ if "pattern" in obj:
71
+ return True
72
+
73
+ # Recursively check all nested objects and arrays
74
+ for value in obj.values():
75
+ if isinstance(value, dict):
76
+ if check_object(value):
77
+ return True
78
+ elif isinstance(value, list):
79
+ for item in value:
80
+ if isinstance(item, dict) and check_object(item):
81
+ return True
82
+
83
+ return False
84
+
85
+ return check_object(schema)
86
+
87
+
88
def maybe_backend_fallback(
89
guided_params: GuidedDecodingParams) -> GuidedDecodingParams:
90
# lm-format-enforce doesn't support grammar, fallback to xgrammar
55
- if (guided_params.backend == "lm-format-enforcer"
56
- and guided_params.grammar is not None):
57
- logger.warning(
58
- "lm-format-enforcer does not support grammar guided decoding. "
59
- "Falling back to use xgrammar instead.")
60
- guided_params.backend = "xgrammar"
91
+ if guided_params.backend == "lm-format-enforcer":
92
+ if guided_params.grammar is not None:
93
+ logger.warning(
94
+ "lm-format-enforcer does not support grammar guided decoding. "
95
+ "Falling back to use xgrammar instead.")
96
+ guided_params.backend = "xgrammar"
97
+
98
+ # lm-format-enforcer doesn't support some JSON schema features
99
+ elif (guided_params.json is not None
100
+ and has_lmf_unsupported_json_features(guided_params.json)):
101
+ logger.warning(
102
+ "lm-format-enforcer does not support advanced JSON schema "
103
+ "features like patterns or numeric ranges. "
104
+ "Falling back to use outlines instead.")
105
+ guided_params.backend = "outlines"
106
107
if guided_params.backend == "xgrammar":
108
# xgrammar only has x86 wheels for linux, fallback to outlines
@@ -82,6 +127,13 @@ def maybe_backend_fallback(
127
"Falling back to use outlines instead.")
128
guided_params.backend = "outlines"
129
130
+ if (guided_params.backend == "outlines"
131
+ and guided_params.json_object is not None):
132
+ # outlines doesn't support json_object, fallback to xgrammar
133
+ logger.warning("outlines does not support json_object. "
134
+ "Falling back to use xgrammar instead.")
135
+ guided_params.backend = "xgrammar"
136
+
137
return guided_params
138
139