@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a30482f054
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[CI] Expand test_guided_generate to test all backends (#11313)

Signed-off-by: mgoin <michael@neuralmagic.com>

Michael Goin committed Dec 18, 2024 at 23:00 UTC a30482f0545a216d6c02241f6663b9cddbcb066e
3 files changed +129 -51
tests/entrypoints/llm/test_guided_generate.py
+69 -43
@@ -10,7 +10,8 @@ from vllm.entrypoints.llm import LLM
10 from vllm.outputs import RequestOutput
11 from vllm.sampling_params import GuidedDecodingParams, SamplingParams
12
13 -MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
13 +MODEL_NAME = "Qwen/Qwen2.5-7B-Instruct"
14 +GUIDED_DECODING_BACKENDS = ["outlines", "lm-format-enforcer", "xgrammar"]
15
16
17 @pytest.fixture(scope="module")
@@ -26,11 +27,13 @@ def llm():
27
28
29 @pytest.mark.skip_global_cleanup
29 -def test_guided_regex(sample_regex, llm):
30 - sampling_params = SamplingParams(
31 - temperature=0.8,
32 - top_p=0.95,
33 - guided_decoding=GuidedDecodingParams(regex=sample_regex))
30 +@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)
31 +def test_guided_regex(sample_regex, llm, guided_decoding_backend: str):
32 + sampling_params = SamplingParams(temperature=0.8,
33 + top_p=0.95,
34 + guided_decoding=GuidedDecodingParams(
35 + regex=sample_regex,
36 + backend=guided_decoding_backend))
37 outputs = llm.generate(prompts=[
38 f"Give an example IPv4 address with this regex: {sample_regex}"
39 ] * 2,
@@ -50,11 +53,14 @@ def test_guided_regex(sample_regex, llm):
53
54
55 @pytest.mark.skip_global_cleanup
53 -def test_guided_json_completion(sample_json_schema, llm):
54 - sampling_params = SamplingParams(
55 - temperature=1.0,
56 - max_tokens=1000,
57 - guided_decoding=GuidedDecodingParams(json=sample_json_schema))
56 +@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)
57 +def test_guided_json_completion(sample_json_schema, llm,
58 + guided_decoding_backend: str):
59 + sampling_params = SamplingParams(temperature=1.0,
60 + max_tokens=1000,
61 + guided_decoding=GuidedDecodingParams(
62 + json=sample_json_schema,
63 + backend=guided_decoding_backend))
64 outputs = llm.generate(prompts=[
65 f"Give an example JSON for an employee profile "
66 f"that fits this schema: {sample_json_schema}"
@@ -77,11 +83,14 @@ def test_guided_json_completion(sample_json_schema, llm):
83
84
85 @pytest.mark.skip_global_cleanup
80 -def test_guided_complex_json_completion(sample_complex_json_schema, llm):
81 - sampling_params = SamplingParams(
82 - temperature=1.0,
83 - max_tokens=1000,
84 - guided_decoding=GuidedDecodingParams(json=sample_complex_json_schema))
86 +@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)
87 +def test_guided_complex_json_completion(sample_complex_json_schema, llm,
88 + guided_decoding_backend: str):
89 + sampling_params = SamplingParams(temperature=1.0,
90 + max_tokens=1000,
91 + guided_decoding=GuidedDecodingParams(
92 + json=sample_complex_json_schema,
93 + backend=guided_decoding_backend))
94 outputs = llm.generate(prompts=[
95 f"Give an example JSON for an assignment grade "
96 f"that fits this schema: {sample_complex_json_schema}"
@@ -105,11 +114,14 @@ def test_guided_complex_json_completion(sample_complex_json_schema, llm):
114
115
116 @pytest.mark.skip_global_cleanup
108 -def test_guided_definition_json_completion(sample_definition_json_schema, llm):
117 +@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)
118 +def test_guided_definition_json_completion(sample_definition_json_schema, llm,
119 + guided_decoding_backend: str):
120 sampling_params = SamplingParams(temperature=1.0,
121 max_tokens=1000,
122 guided_decoding=GuidedDecodingParams(
112 - json=sample_definition_json_schema))
123 + json=sample_definition_json_schema,
124 + backend=guided_decoding_backend))
125 outputs = llm.generate(prompts=[
126 f"Give an example JSON for solving 8x + 7 = -23 "
127 f"that fits this schema: {sample_definition_json_schema}"
@@ -133,11 +145,14 @@ def test_guided_definition_json_completion(sample_definition_json_schema, llm):
145
146
147 @pytest.mark.skip_global_cleanup
136 -def test_guided_choice_completion(sample_guided_choice, llm):
137 - sampling_params = SamplingParams(
138 - temperature=0.8,
139 - top_p=0.95,
140 - guided_decoding=GuidedDecodingParams(choice=sample_guided_choice))
148 +@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)
149 +def test_guided_choice_completion(sample_guided_choice, llm,
150 + guided_decoding_backend: str):
151 + sampling_params = SamplingParams(temperature=0.8,
152 + top_p=0.95,
153 + guided_decoding=GuidedDecodingParams(
154 + choice=sample_guided_choice,
155 + backend=guided_decoding_backend))
156 outputs = llm.generate(
157 prompts="The best language for type-safe systems programming is ",
158 sampling_params=sampling_params,
@@ -156,13 +171,20 @@ def test_guided_choice_completion(sample_guided_choice, llm):
171
172
173 @pytest.mark.skip_global_cleanup
159 -def test_guided_grammar(sample_sql_statements, llm):
160 -
161 - sampling_params = SamplingParams(
162 - temperature=0.8,
163 - top_p=0.95,
164 - max_tokens=1000,
165 - guided_decoding=GuidedDecodingParams(grammar=sample_sql_statements))
174 +@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)
175 +def test_guided_grammar(sample_sql_statements, llm,
176 + guided_decoding_backend: str):
177 + if guided_decoding_backend == "outlines":
178 + pytest.skip("Outlines backend fails in this test case with:\n"
179 + "AttributeError: Error in model execution: 'ParserConf' "
180 + "object has no attribute 'deterministic'")
181 +
182 + sampling_params = SamplingParams(temperature=0.8,
183 + top_p=0.95,
184 + max_tokens=1000,
185 + guided_decoding=GuidedDecodingParams(
186 + grammar=sample_sql_statements,
187 + backend=guided_decoding_backend))
188 outputs = llm.generate(
189 prompts=("Generate a sql state that select col_1 from "
190 "table_1 where it is equals to 1"),
@@ -218,15 +240,18 @@ def test_validation_against_both_guided_decoding_options(sample_regex, llm):
240
241
242 @pytest.mark.skip_global_cleanup
221 -def test_guided_json_object(llm):
222 - sampling_params = SamplingParams(
223 - temperature=1.0,
224 - max_tokens=100,
225 - guided_decoding=GuidedDecodingParams(json_object=True))
243 +@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)
244 +def test_guided_json_object(llm, guided_decoding_backend: str):
245 + sampling_params = SamplingParams(temperature=1.0,
246 + max_tokens=100,
247 + n=2,
248 + guided_decoding=GuidedDecodingParams(
249 + json_object=True,
250 + backend=guided_decoding_backend))
251
252 outputs = llm.generate(
228 - prompts=("Generate a JSON object describing a person with name "
229 - "and age for John Smith who is 31 years old."),
253 + prompts=("Generate a JSON object with curly braces for a person with "
254 + "name and age fields for John Smith who is 31 years old."),
255 sampling_params=sampling_params,
256 use_tqdm=True)
257
@@ -235,10 +260,11 @@ def test_guided_json_object(llm):
260 assert output is not None
261 assert isinstance(output, RequestOutput)
262
238 - generated_text = output.outputs[0].text
239 - print(generated_text)
240 - assert generated_text is not None
263 + for i in range(2):
264 + generated_text = output.outputs[i].text
265 + print(generated_text)
266 + assert generated_text is not None
267
242 - # Parse to verify it is valid JSON
243 - parsed_json = json.loads(generated_text)
244 - assert isinstance(parsed_json, dict)
268 + # Parse to verify it is valid JSON
269 + parsed_json = json.loads(generated_text)
270 + assert isinstance(parsed_json, dict)
tests/model_executor/test_guided_processors.py
+2 -2
@@ -13,6 +13,7 @@ from vllm.model_executor.guided_decoding.outlines_logits_processors import (
13 from vllm.sampling_params import GuidedDecodingParams
14
15 MODEL_NAME = 'HuggingFaceH4/zephyr-7b-beta'
16 +GUIDED_DECODING_BACKENDS = ["outlines", "lm-format-enforcer", "xgrammar"]
17
18
19 def test_guided_logits_processors(sample_regex, sample_json_schema):
@@ -42,8 +43,7 @@ def test_guided_logits_processors(sample_regex, sample_json_schema):
43
44
45 @pytest.mark.asyncio
45 -@pytest.mark.parametrize("backend",
46 - ["outlines", "lm-format-enforcer", "xgrammar"])
46 +@pytest.mark.parametrize("backend", GUIDED_DECODING_BACKENDS)
47 @pytest.mark.parametrize("is_local", [True, False])
48 async def test_guided_logits_processor_black_box(backend: str, is_local: bool,
49 sample_regex,
vllm/model_executor/guided_decoding/__init__.py
+58 -6
@@ -49,15 +49,60 @@ def has_xgrammar_unsupported_json_features(schema: dict) -> bool:
49 return check_object(schema)
50
51
52 +def has_lmf_unsupported_json_features(schema: dict) -> bool:
53 + """
54 + Check if JSON schema contains features unsupported
55 + by lm_format_enforcer.
56 +
57 + Known issues:
58 + - Regex patterns:
59 + "grade": {
60 + "type": "string",
61 + "pattern": "^[A-D]$" # Regex pattern
62 + },
63 + """
64 +
65 + def check_object(obj: dict) -> bool:
66 + if not isinstance(obj, dict):
67 + return False
68 +
69 + # Check for pattern restrictions
70 + if "pattern" in obj:
71 + return True
72 +
73 + # Recursively check all nested objects and arrays
74 + for value in obj.values():
75 + if isinstance(value, dict):
76 + if check_object(value):
77 + return True
78 + elif isinstance(value, list):
79 + for item in value:
80 + if isinstance(item, dict) and check_object(item):
81 + return True
82 +
83 + return False
84 +
85 + return check_object(schema)
86 +
87 +
88 def maybe_backend_fallback(
89 guided_params: GuidedDecodingParams) -> GuidedDecodingParams:
90 # lm-format-enforce doesn't support grammar, fallback to xgrammar
55 - if (guided_params.backend == "lm-format-enforcer"
56 - and guided_params.grammar is not None):
57 - logger.warning(
58 - "lm-format-enforcer does not support grammar guided decoding. "
59 - "Falling back to use xgrammar instead.")
60 - guided_params.backend = "xgrammar"
91 + if guided_params.backend == "lm-format-enforcer":
92 + if guided_params.grammar is not None:
93 + logger.warning(
94 + "lm-format-enforcer does not support grammar guided decoding. "
95 + "Falling back to use xgrammar instead.")
96 + guided_params.backend = "xgrammar"
97 +
98 + # lm-format-enforcer doesn't support some JSON schema features
99 + elif (guided_params.json is not None
100 + and has_lmf_unsupported_json_features(guided_params.json)):
101 + logger.warning(
102 + "lm-format-enforcer does not support advanced JSON schema "
103 + "features like patterns or numeric ranges. "
104 + "Falling back to use outlines instead.")
105 + guided_params.backend = "outlines"
106
107 if guided_params.backend == "xgrammar":
108 # xgrammar only has x86 wheels for linux, fallback to outlines
@@ -82,6 +127,13 @@ def maybe_backend_fallback(
127 "Falling back to use outlines instead.")
128 guided_params.backend = "outlines"
129
130 + if (guided_params.backend == "outlines"
131 + and guided_params.json_object is not None):
132 + # outlines doesn't support json_object, fallback to xgrammar
133 + logger.warning("outlines does not support json_object. "
134 + "Falling back to use xgrammar instead.")
135 + guided_params.backend = "xgrammar"
136 +
137 return guided_params
138
139