@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d6902ce79f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 02:15 UTC

[V0][V1][Core] Add outlines integration for V1, and update V0 integration. (#15975)

Signed-off-by: Nathan Hoos <thwackyy.y@gmail.com>

Nathan Hoos committed Jul 10, 2025 at 14:30 UTC d6902ce79f08b5194a7c73ee435d342e4d4dac8b
13 files changed +807 -464
requirements/common.txt
+3 -1
@@ -21,7 +21,9 @@ prometheus-fastapi-instrumentator >= 7.0.0
21 tiktoken >= 0.6.0 # Required for DBRX tokenizer
22 lm-format-enforcer >= 0.10.11, < 0.11
23 llguidance >= 0.7.11, < 0.8.0; platform_machine == "x86_64" or platform_machine == "arm64" or platform_machine == "aarch64"
24 -outlines == 0.1.11
24 +outlines_core == 0.2.10
25 +# required for outlines backend disk cache
26 +diskcache == 5.6.3
27 lark == 1.2.2
28 xgrammar == 0.1.19; platform_machine == "x86_64" or platform_machine == "aarch64" or platform_machine == "arm64"
29 typing_extensions >= 4.10
tests/entrypoints/llm/test_guided_generate.py
+20 -13
@@ -16,14 +16,18 @@ from vllm.outputs import RequestOutput
16 from vllm.sampling_params import GuidedDecodingParams, SamplingParams
17
18 MODEL_NAME = "Qwen/Qwen2.5-1.5B-Instruct"
19 -GUIDED_DECODING_BACKENDS = [
19 +
20 +# Separate backends which support grammars vs ones
21 +# which only support regex based constraints in tests.
22 +GRAMMAR_DECODING_BACKENDS = [
23 # (backend, disable_any_whitespace),
21 - ("outlines", False),
24 ("lm-format-enforcer", False),
25 ("xgrammar", True),
26 ("guidance", True),
27 ]
28
29 +ALL_DECODING_BACKENDS = ([("outlines", False)] + GRAMMAR_DECODING_BACKENDS)
30 +
31
32 @pytest.fixture(scope="module")
33 def llm():
@@ -39,7 +43,7 @@ def llm():
43
44 @pytest.mark.skip_global_cleanup
45 @pytest.mark.parametrize("guided_decoding_backend,disable_any_whitespace",
42 - GUIDED_DECODING_BACKENDS)
46 + ALL_DECODING_BACKENDS)
47 def test_guided_regex(sample_regex, llm, guided_decoding_backend: str,
48 disable_any_whitespace: bool):
49 sampling_params = SamplingParams(
@@ -49,6 +53,7 @@ def test_guided_regex(sample_regex, llm, guided_decoding_backend: str,
53 regex=sample_regex,
54 backend=guided_decoding_backend,
55 disable_any_whitespace=disable_any_whitespace))
56 +
57 outputs = llm.generate(prompts=[
58 f"Give an example IPv4 address with this regex: {sample_regex}"
59 ] * 2,
@@ -69,7 +74,7 @@ def test_guided_regex(sample_regex, llm, guided_decoding_backend: str,
74
75 @pytest.mark.skip_global_cleanup
76 @pytest.mark.parametrize("guided_decoding_backend,disable_any_whitespace",
72 - GUIDED_DECODING_BACKENDS)
77 + ALL_DECODING_BACKENDS)
78 def test_guided_json_completion(sample_json_schema, llm,
79 guided_decoding_backend: str,
80 disable_any_whitespace: bool):
@@ -103,7 +108,7 @@ def test_guided_json_completion(sample_json_schema, llm,
108
109 @pytest.mark.skip_global_cleanup
110 @pytest.mark.parametrize("guided_decoding_backend,disable_any_whitespace",
106 - GUIDED_DECODING_BACKENDS)
111 + ALL_DECODING_BACKENDS)
112 def test_guided_complex_json_completion(sample_complex_json_schema, llm,
113 guided_decoding_backend: str,
114 disable_any_whitespace: bool):
@@ -138,7 +143,7 @@ def test_guided_complex_json_completion(sample_complex_json_schema, llm,
143
144 @pytest.mark.skip_global_cleanup
145 @pytest.mark.parametrize("guided_decoding_backend,disable_any_whitespace",
141 - GUIDED_DECODING_BACKENDS)
146 + ALL_DECODING_BACKENDS)
147 def test_guided_definition_json_completion(sample_definition_json_schema, llm,
148 guided_decoding_backend: str,
149 disable_any_whitespace: bool):
@@ -173,7 +178,7 @@ def test_guided_definition_json_completion(sample_definition_json_schema, llm,
178
179 @pytest.mark.skip_global_cleanup
180 @pytest.mark.parametrize("guided_decoding_backend,disable_any_whitespace",
176 - GUIDED_DECODING_BACKENDS)
181 + ALL_DECODING_BACKENDS)
182 def test_guided_enum_json_completion(sample_enum_json_schema, llm,
183 guided_decoding_backend: str,
184 disable_any_whitespace: bool):
@@ -218,7 +223,7 @@ def test_guided_enum_json_completion(sample_enum_json_schema, llm,
223
224 @pytest.mark.skip_global_cleanup
225 @pytest.mark.parametrize("guided_decoding_backend,disable_any_whitespace",
221 - GUIDED_DECODING_BACKENDS)
226 + ALL_DECODING_BACKENDS)
227 def test_guided_choice_completion(sample_guided_choice, llm,
228 guided_decoding_backend: str,
229 disable_any_whitespace: bool):
@@ -248,7 +253,7 @@ def test_guided_choice_completion(sample_guided_choice, llm,
253
254 @pytest.mark.skip_global_cleanup
255 @pytest.mark.parametrize("guided_decoding_backend,disable_any_whitespace",
251 - GUIDED_DECODING_BACKENDS)
256 + GRAMMAR_DECODING_BACKENDS)
257 def test_guided_grammar(sample_sql_statements, llm,
258 guided_decoding_backend: str,
259 disable_any_whitespace: bool):
@@ -344,7 +349,7 @@ def test_disable_guided_decoding_fallback(sample_regex, llm):
349
350 @pytest.mark.skip_global_cleanup
351 @pytest.mark.parametrize("guided_decoding_backend,disable_any_whitespace",
347 - GUIDED_DECODING_BACKENDS)
352 + GRAMMAR_DECODING_BACKENDS)
353 def test_guided_json_object(llm, guided_decoding_backend: str,
354 disable_any_whitespace: bool):
355 sampling_params = SamplingParams(
@@ -377,7 +382,9 @@ def test_guided_json_object(llm, guided_decoding_backend: str,
382
383 # Parse to verify it is valid JSON
384 parsed_json = json.loads(generated_text)
380 - assert isinstance(parsed_json, dict)
385 + # A list is not what was intended, but is still valid
386 + # json.
387 + assert isinstance(parsed_json, (dict, list))
388
389
390 class CarType(str, Enum):
@@ -395,7 +402,7 @@ class CarDescription(BaseModel):
402
403 @pytest.mark.skip_global_cleanup
404 @pytest.mark.parametrize("guided_decoding_backend,disable_any_whitespace",
398 - GUIDED_DECODING_BACKENDS)
405 + ALL_DECODING_BACKENDS)
406 def test_guided_json_completion_with_enum(llm, guided_decoding_backend: str,
407 disable_any_whitespace: bool):
408 json_schema = CarDescription.model_json_schema()
@@ -427,7 +434,7 @@ def test_guided_json_completion_with_enum(llm, guided_decoding_backend: str,
434
435 @pytest.mark.skip_global_cleanup
436 @pytest.mark.parametrize("guided_decoding_backend,disable_any_whitespace",
430 - GUIDED_DECODING_BACKENDS)
437 + ALL_DECODING_BACKENDS)
438 def test_guided_number_range_json_completion(llm, guided_decoding_backend: str,
439 disable_any_whitespace: bool):
440 sample_output_schema = {
tests/model_executor/test_guided_processors.py
+9 -21
@@ -46,20 +46,15 @@ def test_guided_logits_processors(zephyr_7B_tokenzer, sample_regex,
46 whitespace_pattern=None,
47 reasoner=None)
48
49 - token_ids = zephyr_7B_tokenzer.encode(
50 - f"Give an example IPv4 address with this regex: {sample_regex}")
49 tensor = torch.rand(32000)
50 original_tensor = torch.clone(tensor)
53 - regex_LP(token_ids, tensor)
51 + tensor = regex_LP([], tensor)
52 assert tensor.shape == original_tensor.shape
53 assert not torch.allclose(tensor, original_tensor)
54
57 - token_ids = zephyr_7B_tokenzer.encode(
58 - f"Give an employee profile that fits this schema: {sample_json_schema}"
59 - )
55 tensor = torch.rand(32000)
56 original_tensor = torch.clone(tensor)
62 - json_LP(token_ids, tensor)
57 + tensor = json_LP([], tensor)
58 assert tensor.shape == original_tensor.shape
59 assert not torch.allclose(tensor, original_tensor)
60
@@ -81,8 +76,6 @@ async def test_guided_logits_processor_black_box(backend: str, is_local: bool,
76 seed=0,
77 dtype="bfloat16",
78 )
84 - token_ids = zephyr_7B_tokenzer.encode(
85 - f"Give an example IPv4 address with this regex: {sample_regex}")
79 regex_request = GuidedDecodingParams(regex=sample_regex, backend=backend)
80
81 regex_lp = get_local_guided_decoding_logits_processor(
@@ -92,13 +85,11 @@ async def test_guided_logits_processor_black_box(backend: str, is_local: bool,
85 assert regex_lp is not None
86 tensor = torch.rand(32000)
87 original_tensor = torch.clone(tensor)
95 - tensor = regex_lp(token_ids, tensor)
88 + # allowed tokens at state 0
89 + tensor = regex_lp([], tensor)
90 assert tensor.shape == original_tensor.shape
91 assert not torch.allclose(tensor, original_tensor)
92
99 - token_ids = zephyr_7B_tokenzer.encode(
100 - f"Give an employee profile that fits this schema: {sample_json_schema}"
101 - )
93 json_request = GuidedDecodingParams(json=sample_json_schema,
94 backend=backend)
95 json_lp = await get_guided_decoding_logits_processor(
@@ -106,7 +97,7 @@ async def test_guided_logits_processor_black_box(backend: str, is_local: bool,
97 assert json_lp is not None
98 tensor = torch.rand(32000)
99 original_tensor = torch.clone(tensor)
109 - tensor = json_lp(token_ids, tensor)
100 + tensor = json_lp([], tensor)
101 assert tensor.shape == original_tensor.shape
102 assert not torch.allclose(tensor, original_tensor)
103
@@ -130,7 +121,6 @@ async def test_guided_logits_processor_with_reasoning(
121 dtype="bfloat16",
122 )
123 token_ids = deepseek_r1_qwen_tokenizer.encode(
133 - f"Give an example IPv4 address with this regex: {sample_regex}."
124 "<think>here is the thinking process")
125 regex_request = GuidedDecodingParams(regex=sample_regex, backend=backend)
126
@@ -141,14 +131,13 @@ async def test_guided_logits_processor_with_reasoning(
131 regex_request, deepseek_r1_qwen_tokenizer, config,
132 reasoning_backend)
133 assert regex_lp is not None
144 - tensor = torch.rand(32000)
134 + tensor = torch.rand(151664)
135 original_tensor = torch.clone(tensor)
136 tensor = regex_lp(token_ids, tensor)
137 assert tensor.shape == original_tensor.shape
138 assert torch.allclose(tensor, original_tensor)
139
140 token_ids = deepseek_r1_qwen_tokenizer.encode(
151 - f"Give an employee profile that fits this schema: {sample_json_schema}."
141 "<think>here is the thinking process")
142 json_request = GuidedDecodingParams(json=sample_json_schema,
143 backend=backend)
@@ -158,7 +147,7 @@ async def test_guided_logits_processor_with_reasoning(
147 await get_guided_decoding_logits_processor(
148 json_request, deepseek_r1_qwen_tokenizer, config, reasoning_backend)
149 assert json_lp is not None
161 - tensor = torch.rand(32000)
150 + tensor = torch.rand(151664)
151 original_tensor = torch.clone(tensor)
152 tensor = json_lp(token_ids, tensor)
153 assert tensor.shape == original_tensor.shape
@@ -166,8 +155,7 @@ async def test_guided_logits_processor_with_reasoning(
155
156 # Thinking is over, so the tensor should change.
157 token_ids = deepseek_r1_qwen_tokenizer.encode(
169 - f"Give an employee profile that fits this schema: {sample_json_schema}."
170 - "<think>here is the thinking process</think> Then")
158 + "<think>here is the thinking process</think>")
159 json_request = GuidedDecodingParams(json=sample_json_schema,
160 backend=backend)
161 json_lp = get_local_guided_decoding_logits_processor(
@@ -176,7 +164,7 @@ async def test_guided_logits_processor_with_reasoning(
164 await get_guided_decoding_logits_processor(
165 json_request, deepseek_r1_qwen_tokenizer, config, reasoning_backend)
166 assert json_lp is not None
179 - tensor = torch.rand(32000)
167 + tensor = torch.rand(151664)
168 original_tensor = torch.clone(tensor)
169 tensor = json_lp(token_ids, tensor)
170 assert tensor.shape == original_tensor.shape
tests/tool_use/test_tool_choice_required.py
+1 -1
@@ -72,7 +72,7 @@ def _compile_and_check(tools: list[ChatCompletionToolsParam], sample_output,
72 assert isinstance(schema, dict)
73
74 # use build_regex_from_schema used in JSONLogitsProcessor to create Guide
75 - from outlines_core.fsm.json_schema import build_regex_from_schema
75 + from outlines_core.json_schema import build_regex_from_schema
76 regex = build_regex_from_schema(json.dumps(schema))
77 compiled = re.compile(regex)
78 matches = compiled.fullmatch(json.dumps(sample_output)) is not None
tests/v1/entrypoints/llm/test_struct_output_generate.py
+171 -162
@@ -41,6 +41,10 @@ PARAMS_MODELS_BACKENDS_TOKENIZER_MODE = [
41 ("mistralai/Ministral-8B-Instruct-2410", "guidance", "auto", None),
42 ("mistralai/Ministral-8B-Instruct-2410", "xgrammar", "mistral", None),
43 ("Qwen/Qwen2.5-1.5B-Instruct", "xgrammar", "auto", None),
44 + ("mistralai/Ministral-8B-Instruct-2410", "outlines", "auto", None),
45 + ("mistralai/Ministral-8B-Instruct-2410", "outlines", "mistral", None),
46 + ("mistralai/Ministral-8B-Instruct-2410", "outlines", "auto",
47 + NGRAM_SPEC_CONFIG),
48 #FIXME: This test is flaky on CI thus disabled
49 #("Qwen/Qwen2.5-1.5B-Instruct", "guidance", "auto"),
50 ("mistralai/Ministral-8B-Instruct-2410", "guidance", "auto",
@@ -106,13 +110,15 @@ def test_structured_output(
110 enforce_eager = bool(not current_platform.is_tpu())
111 # Use a single LLM instance for several scenarios to
112 # speed up the test suite.
109 - llm = LLM(model=model_name,
110 - enforce_eager=enforce_eager,
111 - max_model_len=1024,
112 - guided_decoding_backend=guided_decoding_backend,
113 - guided_decoding_disable_any_whitespace=True,
114 - tokenizer_mode=tokenizer_mode,
115 - speculative_config=speculative_config)
113 + llm = LLM(
114 + model=model_name,
115 + enforce_eager=enforce_eager,
116 + max_model_len=1024,
117 + guided_decoding_backend=guided_decoding_backend,
118 + guided_decoding_disable_any_whitespace=(guided_decoding_backend
119 + in {"xgrammar", "guidance"}),
120 + tokenizer_mode=tokenizer_mode,
121 + speculative_config=speculative_config)
122
123 #
124 # Test 1: Generate JSON output based on a provided schema
@@ -146,32 +152,33 @@ def test_structured_output(
152 #
153 # Test 2: Generate JSON object without a schema
154 #
149 - sampling_params = SamplingParams(
150 - temperature=1.0,
151 - max_tokens=4096,
152 - n=2,
153 - guided_decoding=GuidedDecodingParams(json_object=True))
155 + if guided_decoding_backend != "outlines":
156 + sampling_params = SamplingParams(
157 + temperature=1.0,
158 + max_tokens=4096,
159 + n=2,
160 + guided_decoding=GuidedDecodingParams(json_object=True))
161
155 - outputs = llm.generate(
156 - prompts=("Generate a JSON object with curly braces for a person with "
157 - "name and age fields for John Smith who is 31 years old. "
158 - "Make the response as short as possible."),
159 - sampling_params=sampling_params,
160 - use_tqdm=True)
162 + outputs = llm.generate(prompts=(
163 + "Generate a JSON object with curly braces for a person with "
164 + "name and age fields for John Smith who is 31 years old. "
165 + "Make the response as short as possible."),
166 + sampling_params=sampling_params,
167 + use_tqdm=True)
168
162 - assert outputs is not None
163 - for output in outputs:
164 - assert output is not None
165 - assert isinstance(output, RequestOutput)
169 + assert outputs is not None
170 + for output in outputs:
171 + assert output is not None
172 + assert isinstance(output, RequestOutput)
173
167 - for i in range(2):
168 - generated_text = output.outputs[i].text
169 - print(generated_text)
170 - assert generated_text is not None
174 + for i in range(2):
175 + generated_text = output.outputs[i].text
176 + print(generated_text)
177 + assert generated_text is not None
178
172 - # Parse to verify it is a valid JSON object
173 - parsed_json = json.loads(generated_text)
174 - assert isinstance(parsed_json, dict)
179 + # Parse to verify it is a valid JSON object
180 + parsed_json = json.loads(generated_text)
181 + assert isinstance(parsed_json, dict)
182
183 #
184 # Test 3: test a jsonschema incompatible with xgrammar
@@ -210,96 +217,97 @@ def test_structured_output(
217 parsed_json = json.loads(generated_text)
218 assert isinstance(parsed_json, dict)
219
213 - #
214 - # Test 4: Generate SQL statement using EBNF grammar
215 - #
216 - sampling_params = SamplingParams(
217 - temperature=0.8,
218 - top_p=0.95,
219 - max_tokens=1000,
220 - guided_decoding=GuidedDecodingParams(grammar=sample_sql_ebnf))
221 - outputs = llm.generate(
222 - prompts=(
223 - "Generate a sql statement that selects col_1 from "
224 - "table_1 where it is equal to 1. Make the response as short as "
225 - "possible."),
226 - sampling_params=sampling_params,
227 - use_tqdm=True,
228 - )
220 + if guided_decoding_backend != "outlines":
221 + #
222 + # Test 4: Generate SQL statement using EBNF grammar
223 + #
224 + sampling_params = SamplingParams(
225 + temperature=0.8,
226 + top_p=0.95,
227 + max_tokens=1000,
228 + guided_decoding=GuidedDecodingParams(grammar=sample_sql_ebnf))
229 + outputs = llm.generate(
230 + prompts=(
231 + "Generate a sql statement that selects col_1 from "
232 + "table_1 where it is equal to 1. Make the response as short as "
233 + "possible."),
234 + sampling_params=sampling_params,
235 + use_tqdm=True,
236 + )
237
230 - assert outputs is not None
231 - for output in outputs:
232 - assert output is not None
233 - assert isinstance(output, RequestOutput)
234 - prompt = output.prompt
238 + assert outputs is not None
239 + for output in outputs:
240 + assert output is not None
241 + assert isinstance(output, RequestOutput)
242 + prompt = output.prompt
243
236 - generated_text = output.outputs[0].text
237 - assert generated_text is not None
244 + generated_text = output.outputs[0].text
245 + assert generated_text is not None
246
239 - # remove spaces for comparison b/c we removed them in the grammar
240 - ground_truth = "SELECT col_1 from table_1 where col_1 = 1".replace(
241 - " ", "")
247 + # remove spaces for comparison b/c we removed them in the grammar
248 + ground_truth = "SELECT col_1 from table_1 where col_1 = 1".replace(
249 + " ", "")
250
243 - assert generated_text.strip() == ground_truth
251 + assert generated_text.strip() == ground_truth
252
245 - print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
253 + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
254
247 - #
248 - # Test 5: Generate SQL statement using Lark grammar
249 - #
250 - sampling_params = SamplingParams(
251 - temperature=0.8,
252 - top_p=0.95,
253 - max_tokens=1000,
254 - guided_decoding=GuidedDecodingParams(grammar=sample_sql_lark))
255 - outputs = llm.generate(
256 - prompts=(
257 - "Generate a sql statement that selects col_1 from "
258 - "table_1 where it is equal to 1. Make the response as short as "
259 - "possible."),
260 - sampling_params=sampling_params,
261 - use_tqdm=True,
262 - )
255 + #
256 + # Test 5: Generate SQL statement using Lark grammar
257 + #
258 + sampling_params = SamplingParams(
259 + temperature=0.8,
260 + top_p=0.95,
261 + max_tokens=1000,
262 + guided_decoding=GuidedDecodingParams(grammar=sample_sql_lark))
263 + outputs = llm.generate(
264 + prompts=(
265 + "Generate a sql statement that selects col_1 from "
266 + "table_1 where it is equal to 1. Make the response as short as "
267 + "possible."),
268 + sampling_params=sampling_params,
269 + use_tqdm=True,
270 + )
271
264 - assert outputs is not None
265 - for output in outputs:
266 - assert output is not None
267 - assert isinstance(output, RequestOutput)
268 - prompt = output.prompt
272 + assert outputs is not None
273 + for output in outputs:
274 + assert output is not None
275 + assert isinstance(output, RequestOutput)
276 + prompt = output.prompt
277
270 - generated_text = output.outputs[0].text
271 - assert generated_text is not None
278 + generated_text = output.outputs[0].text
279 + assert generated_text is not None
280
273 - # use Lark to parse the output, and make sure it's a valid parse tree
274 - from lark import Lark
275 - parser = Lark(sample_sql_lark)
276 - parser.parse(generated_text)
281 + # use Lark to parse the output, and make sure it's a valid parse tree
282 + from lark import Lark
283 + parser = Lark(sample_sql_lark)
284 + parser.parse(generated_text)
285
278 - # remove spaces for comparison b/c we removed them in the grammar
279 - ground_truth = "SELECT col_1 from table_1 where col_1 = 1".replace(
280 - " ", "")
286 + # remove spaces for comparison b/c we removed them in the grammar
287 + ground_truth = "SELECT col_1 from table_1 where col_1 = 1".replace(
288 + " ", "")
289
282 - assert generated_text.strip() == ground_truth
290 + assert generated_text.strip() == ground_truth
291
284 - print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
292 + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
293
286 - #
287 - # Test 6: Test invalid grammar input
288 - #
289 - sampling_params = SamplingParams(
290 - temperature=0.8,
291 - top_p=0.95,
292 - max_tokens=1000,
293 - guided_decoding=GuidedDecodingParams(grammar="not a grammar"))
294 - with pytest.raises(ValueError, match="Failed to convert the grammar "):
295 - llm.generate(
296 - prompts=(
297 - "Generate a sql statement that selects col_1 from "
298 - "table_1 where it is equal to 1. Make the response as short "
299 - "as possible."),
300 - sampling_params=sampling_params,
301 - use_tqdm=True,
302 - )
294 + #
295 + # Test 6: Test invalid grammar input
296 + #
297 + sampling_params = SamplingParams(
298 + temperature=0.8,
299 + top_p=0.95,
300 + max_tokens=1000,
301 + guided_decoding=GuidedDecodingParams(grammar="not a grammar"))
302 + with pytest.raises(ValueError, match="Failed to convert the grammar "):
303 + llm.generate(
304 + prompts=
305 + ("Generate a sql statement that selects col_1 from "
306 + "table_1 where it is equal to 1. Make the response as short "
307 + "as possible."),
308 + sampling_params=sampling_params,
309 + use_tqdm=True,
310 + )
311
312 #
313 # Test 7: Generate text based on a regex pattern
@@ -421,35 +429,36 @@ def test_structured_output(
429 output_json = json.loads(generated_text)
430 jsonschema.validate(instance=output_json, schema=json_schema)
431
424 - #
425 - # Test 11: Generate structured output using structural_tag format
426 - #
427 - structural_tag_config = {
428 - "type":
429 - "structural_tag",
430 - "structures": [{
431 - "begin": "<function=get_weather>",
432 - "schema": {
433 - "type": "object",
434 - "properties": {
435 - "city": {
436 - "type": "string"
437 - }
432 + if guided_decoding_backend != "outlines":
433 + #
434 + # Test 11: Generate structured output using structural_tag format
435 + #
436 + structural_tag_config = {
437 + "type":
438 + "structural_tag",
439 + "structures": [{
440 + "begin": "<function=get_weather>",
441 + "schema": {
442 + "type": "object",
443 + "properties": {
444 + "city": {
445 + "type": "string"
446 + }
447 + },
448 + "additionalProperties": False
449 },
439 - "additionalProperties": False
440 - },
441 - "end": "</function>"
442 - }],
443 - "triggers": ["<function="]
444 - }
450 + "end": "</function>"
451 + }],
452 + "triggers": ["<function="]
453 + }
454
446 - sampling_params = SamplingParams(
447 - temperature=0.0,
448 - max_tokens=4096,
449 - guided_decoding=GuidedDecodingParams(
450 - structural_tag=json.dumps(structural_tag_config)))
455 + sampling_params = SamplingParams(
456 + temperature=0.0,
457 + max_tokens=4096,
458 + guided_decoding=GuidedDecodingParams(
459 + structural_tag=json.dumps(structural_tag_config)))
460
452 - prompt = """
461 + prompt = """
462 You have access to the following function to retrieve the weather in a city:
463
464 {
@@ -469,7 +478,7 @@ where
478
479 start_tag => `<function`
480 parameters => a JSON dict with the function argument name
472 - as key and function argument value as value.
481 + as key and function argument value as value.
482 end_tag => `</function>`
483
484 Here is an example,
@@ -488,37 +497,37 @@ Given the previous instructions, what is the weather in New York City? \
497 Make the response as short as possible.
498 """
499
491 - # Change this once other backends support structural_tag
492 - outputs = llm.generate(prompts=prompt,
493 - sampling_params=sampling_params,
494 - use_tqdm=True)
495 - assert outputs is not None
500 + # Change this once other backends support structural_tag
501 + outputs = llm.generate(prompts=prompt,
502 + sampling_params=sampling_params,
503 + use_tqdm=True)
504 + assert outputs is not None
505
497 - for output in outputs:
498 - assert output is not None
499 - assert isinstance(output, RequestOutput)
500 - generated_text = output.outputs[0].text
501 - assert generated_text is not None
506 + for output in outputs:
507 + assert output is not None
508 + assert isinstance(output, RequestOutput)
509 + generated_text = output.outputs[0].text
510 + assert generated_text is not None
511
503 - # Search for function call pattern in the response
504 - function_call_pattern = r'<function=get_weather>(.*?)</function>'
505 - matches = re.findall(function_call_pattern, generated_text)
506 -
507 - if not matches:
508 - print(f"Warning: No function calls found in response: "
509 - f"{generated_text!r}")
510 - continue
511 -
512 - # Take the first function call if multiple are found
513 - json_str = matches[0]
514 - try:
515 - json_content = json.loads(json_str)
516 - assert "city" in json_content
517 - assert isinstance(json_content["city"], str)
518 - print(f"Found valid function call: {generated_text!r}")
519 - except (json.JSONDecodeError, AssertionError) as e:
520 - pytest.fail("Invalid function call format: "
521 - f"{generated_text!r}\nError: {str(e)}")
512 + # Search for function call pattern in the response
513 + function_call_pattern = r'<function=get_weather>(.*?)</function>'
514 + matches = re.findall(function_call_pattern, generated_text)
515 +
516 + if not matches:
517 + print(f"Warning: No function calls found in response: "
518 + f"{generated_text!r}")
519 + continue
520 +
521 + # Take the first function call if multiple are found
522 + json_str = matches[0]
523 + try:
524 + json_content = json.loads(json_str)
525 + assert "city" in json_content
526 + assert isinstance(json_content["city"], str)
527 + print(f"Found valid function call: {generated_text!r}")
528 + except (json.JSONDecodeError, AssertionError) as e:
529 + pytest.fail("Invalid function call format: "
530 + f"{generated_text!r}\nError: {str(e)}")
531
532
533 @pytest.mark.skip_global_cleanup
vllm/config.py
+2 -1
@@ -3580,7 +3580,8 @@ def get_served_model_name(model: str,
3580
3581 GuidedDecodingBackendV0 = Literal["auto", "outlines", "lm-format-enforcer",
3582 "xgrammar", "guidance"]
3583 -GuidedDecodingBackendV1 = Literal["auto", "xgrammar", "guidance"]
3583 +
3584 +GuidedDecodingBackendV1 = Literal["auto", "xgrammar", "guidance", "outlines"]
3585 GuidedDecodingBackend = Literal[GuidedDecodingBackendV0,
3586 GuidedDecodingBackendV1]
3587
vllm/envs.py
+7
@@ -117,6 +117,7 @@ if TYPE_CHECKING:
117 VLLM_RANDOMIZE_DP_DUMMY_INPUTS: bool = False
118 VLLM_MARLIN_USE_ATOMIC_ADD: bool = False
119 VLLM_V0_USE_OUTLINES_CACHE: bool = False
120 + VLLM_V1_USE_OUTLINES_CACHE: bool = False
121 VLLM_TPU_BUCKET_PADDING_GAP: int = 0
122 VLLM_TPU_MOST_MODEL_LEN: Optional[int] = None
123 VLLM_USE_DEEP_GEMM: bool = False
@@ -847,6 +848,12 @@ environment_variables: dict[str, Callable[[], Any]] = {
848 "VLLM_V0_USE_OUTLINES_CACHE":
849 lambda: os.environ.get("VLLM_V0_USE_OUTLINES_CACHE", "0") == "1",
850
851 + # Whether to turn on the outlines cache for V1
852 + # This cache is unbounded and on disk, so it's not safe to use in
853 + # an environment with potentially malicious users.
854 + "VLLM_V1_USE_OUTLINES_CACHE":
855 + lambda: os.environ.get("VLLM_V1_USE_OUTLINES_CACHE", "0") == "1",
856 +
857 # Gap between padding buckets for the forward pass. So we have
858 # 8, we will run forward pass with [16, 24, 32, ...].
859 "VLLM_TPU_BUCKET_PADDING_GAP":
vllm/model_executor/guided_decoding/__init__.py
+21 -10
@@ -79,20 +79,33 @@ def maybe_backend_fallback(
79 fallback_or_error(
80 guided_params,
81 "xgrammar does not support Lark grammars and the "
82 - "grammar failed to convert to GBNF.", "outlines")
82 + "grammar failed to convert to GBNF.", "guidance")
83
84 # If the xgrammar module cannot be imported successfully,
85 # we should still allow users to use guided decoding with a fallback.
86 elif not xgr_installed:
87 fallback_or_error(
88 guided_params,
89 - "xgrammar module cannot be imported successfully.", "outlines")
90 -
91 - if (guided_params.backend == "outlines"
92 - and guided_params.json_object is not None):
93 - # outlines doesn't support json_object, fallback to guidance
94 - fallback_or_error(guided_params,
95 - "outlines does not support json_object.", "guidance")
89 + "xgrammar module cannot be imported successfully.", "guidance")
90 +
91 + if guided_params.backend == "outlines":
92 + if guided_params.json_object is not None:
93 + # outlines doesn't support json_object, fallback to guidance
94 + fallback_or_error(guided_params,
95 + "outlines does not support json_object.",
96 + "guidance")
97 + elif guided_params.grammar is not None:
98 + # outlines grammar support has been removed, fallback to guidance
99 + # if it is a lark-based grammar and xgrammar otherwise
100 + if grammar_is_likely_lark(guided_params.grammar):
101 + fallback_or_error(guided_params,
102 + "outlines no longer supports grammars.",
103 + "guidance")
104 + else:
105 + # The grammar is likely already GBNF format.
106 + fallback_or_error(guided_params,
107 + "outlines no longer supports grammars.",
108 + "xgrammar")
109
110 return guided_params
111
@@ -111,7 +124,6 @@ async def get_guided_decoding_logits_processor(
124
125 guided_params = maybe_backend_fallback(guided_params)
126
114 - # CFG grammar not supported by LMFE, so we use outlines instead
127 if guided_params.backend == 'outlines':
128 # NOTE: lazy import outlines to avoid https://github.com/vllm-project/vllm/issues/4193
129 from vllm.model_executor.guided_decoding.outlines_decoding import ( # noqa
@@ -152,7 +164,6 @@ def get_local_guided_decoding_logits_processor(
164 reasoning_backend)
165 reasoner = reasoner_class(tokenizer)
166
155 - # CFG grammar not supported by LMFE, so we use outlines instead
167 if guided_params.backend == 'outlines':
168 # NOTE: lazy import outlines to avoid https://github.com/vllm-project/vllm/issues/4193
169 from vllm.model_executor.guided_decoding.outlines_decoding import ( # noqa
vllm/model_executor/guided_decoding/outlines_decoding.py
+12 -50
@@ -12,7 +12,7 @@ from regex import escape as regex_escape
12 from transformers import PreTrainedTokenizerBase
13
14 from vllm.model_executor.guided_decoding.outlines_logits_processors import (
15 - CFGLogitsProcessor, JSONLogitsProcessor, RegexLogitsProcessor)
15 + JSONLogitsProcessor, RegexLogitsProcessor)
16 from vllm.reasoning import ReasoningParser
17 from vllm.sampling_params import GuidedDecodingParams
18
@@ -21,36 +21,8 @@ class GuidedDecodingMode(Enum):
21 JSON = "json"
22 REGEX = "regex"
23 CHOICE = "choice"
24 - GRAMMAR = "grammar"
24
25
27 -# https://github.com/outlines-dev/outlines/blob/main/outlines/grammars/json.lark
28 -# the main difference is that we changed the start: value to
29 -# start: object | array, so we are denying scalar values as the root of the
30 -# JSON. Starting with scalars as the root seems to cause llama to generate
31 -# without stop.
32 -JSON_GRAMMAR = r"""
33 -?start: object | array
34 -
35 -?value: object
36 -| array
37 -| UNESCAPED_STRING
38 -| SIGNED_NUMBER -> number
39 -| "true" -> true
40 -| "false" -> false
41 -| "null" -> null
42 -
43 -array : "[" [value ("," value)*] "]"
44 -object : "{" [pair ("," pair)*] "}"
45 -pair : UNESCAPED_STRING ":" value
46 -
47 -%import common.UNESCAPED_STRING
48 -%import common.SIGNED_NUMBER
49 -%import common.WS
50 -
51 -%ignore WS
52 -"""
53 -
26 global_thread_pool = None # used for generating logits processor fsm
27
28 # It's not yet clear that using more provides a benefit, and it could
@@ -60,16 +32,12 @@ _MAX_THREADPOOL_WORKERS = 16
32
33
34 async def get_outlines_guided_decoding_logits_processor(
63 - guided_params: GuidedDecodingParams,
64 - tokenizer: PreTrainedTokenizerBase,
65 - reasoner: Optional[ReasoningParser],
66 -) -> Union[JSONLogitsProcessor, RegexLogitsProcessor, CFGLogitsProcessor,
67 - None]:
35 + guided_params: GuidedDecodingParams, tokenizer: PreTrainedTokenizerBase,
36 + reasoner: Optional[ReasoningParser]
37 +) -> Union[JSONLogitsProcessor, RegexLogitsProcessor, None]:
38 """
39 Given an OpenAI-compatible request, check for guided decoding parameters
40 and get the necessary logits processor for the given guide.
71 - We cache logit processors by (guide, tokenizer), and on cache hit
72 - we make a shallow copy to reuse the same underlying FSM.
41 """
42 global global_thread_pool
43 guide, mode = _get_guide_and_mode(guided_params)
@@ -83,7 +51,6 @@ async def get_outlines_guided_decoding_logits_processor(
51 global_thread_pool = concurrent.futures.ThreadPoolExecutor(
52 max_workers=max_workers)
53 loop = asyncio.get_running_loop()
86 -
54 return await loop.run_in_executor(global_thread_pool,
55 _get_logits_processor, guide, tokenizer,
56 mode, guided_params.whitespace_pattern,
@@ -91,16 +58,12 @@ async def get_outlines_guided_decoding_logits_processor(
58
59
60 def get_local_outlines_guided_decoding_logits_processor(
94 - guided_params: GuidedDecodingParams,
95 - tokenizer: PreTrainedTokenizerBase,
96 - reasoner: Optional[ReasoningParser],
97 -) -> Union[JSONLogitsProcessor, RegexLogitsProcessor, CFGLogitsProcessor,
98 - None]:
61 + guided_params: GuidedDecodingParams, tokenizer: PreTrainedTokenizerBase,
62 + reasoner: Optional[ReasoningParser]
63 +) -> Union[JSONLogitsProcessor, RegexLogitsProcessor, None]:
64 """
65 Given an OpenAI-compatible request, check for guided decoding parameters
66 and get the necessary logits processor for the given guide.
102 - We cache logit processors by (guide, tokenizer), and on cache hit
103 - we make a shallow copy to reuse the same underlying FSM.
67 """
68 guide, mode = _get_guide_and_mode(guided_params)
69 if not guide or not mode:
@@ -130,9 +93,10 @@ def _get_guide_and_mode(
93 choices_regex = "(" + "|".join(choices) + ")"
94 return choices_regex, GuidedDecodingMode.CHOICE
95 elif guided_params.grammar:
133 - return guided_params.grammar, GuidedDecodingMode.GRAMMAR
134 - elif guided_params.json_object:
135 - return JSON_GRAMMAR, GuidedDecodingMode.GRAMMAR
96 + raise ValueError(
97 + "The `outlines` guided decoding backend no longer supports grammar "
98 + "guided generation. Please use either the `xgrammar` or `guidance` "
99 + "backend")
100 else:
101 return None, None
102
@@ -143,13 +107,11 @@ def _get_logits_processor(
107 mode: GuidedDecodingMode,
108 whitespace_pattern: Union[str, None],
109 reasoner: Optional[ReasoningParser],
146 -) -> Union[JSONLogitsProcessor, RegexLogitsProcessor, CFGLogitsProcessor]:
110 +) -> Union[JSONLogitsProcessor, RegexLogitsProcessor]:
111 if mode == GuidedDecodingMode.JSON:
112 return JSONLogitsProcessor(guide, tokenizer, whitespace_pattern,
113 reasoner)
114 elif mode == GuidedDecodingMode.REGEX or mode == GuidedDecodingMode.CHOICE:
115 return RegexLogitsProcessor(guide, tokenizer, reasoner)
152 - elif mode == GuidedDecodingMode.GRAMMAR:
153 - return CFGLogitsProcessor(guide, tokenizer, reasoner)
116 else:
117 raise ValueError(f"Unknown guided decoding mode {mode}")
vllm/model_executor/guided_decoding/outlines_logits_processors.py
+228 -205
@@ -1,168 +1,124 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +# SPDX-FileCopyrightText: Copyright 2024-present the Outlines developers
4 +from __future__ import annotations
5
4 -# Copyright 2024- the Outlines developers
5 -# This file is adapted from
6 -# https://github.com/outlines-dev/outlines/blob/main/outlines/serve/vllm.py
7 -#
8 -# Licensed under the Apache License, Version 2.0 (the "License");
9 -# you may not use this file except in compliance with the License.
10 -# You may obtain a copy of the License at
11 -
12 -# http://www.apache.org/licenses/LICENSE-2.0
13 -
14 -# Unless required by applicable law or agreed to in writing, software
15 -# distributed under the License is distributed on an "AS IS" BASIS,
16 -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17 -# See the License for the specific language governing permissions and
18 -# limitations under the License.
6 import copy
7 +import hashlib
8 +import importlib.metadata
9 import json
21 -from collections import defaultdict
22 -from functools import lru_cache
23 -from typing import Callable, Optional, Union
10 +import os
11 +from typing import Optional, Union
12
25 -import numpy as np
13 +import regex as re
14 import torch
27 -from outlines import grammars
28 -from outlines.caching import cache, disable_cache
29 -from outlines.fsm.guide import (CFGGuide, CFGState, Generate, Guide,
30 - RegexGuide, Write)
31 -from outlines.fsm.parsing import PartialLark
32 -from outlines_core.fsm.json_schema import build_regex_from_schema
15 +from cachetools import LRUCache
16 +from diskcache import Cache
17 +from outlines_core import Guide, Index, Vocabulary
18 +from outlines_core.json_schema import build_regex_from_schema
19 +from outlines_core.kernels.torch import (_apply_token_bitmask_inplace_kernel,
20 + allocate_token_bitmask)
21 from pydantic import BaseModel
22 from transformers import PreTrainedTokenizerBase
23 +from transformers.file_utils import SPIECE_UNDERLINE
24 +from transformers.models.gpt2.tokenization_gpt2 import bytes_to_unicode
25
26 import vllm.envs as envs
27 from vllm.logger import init_logger
38 -from vllm.platforms import current_platform
28 from vllm.reasoning import ReasoningParser
29 +from vllm.transformers_utils.tokenizer import AnyTokenizer
30
31 logger = init_logger(__name__)
32
43 -if envs.VLLM_V0_USE_OUTLINES_CACHE:
44 - logger.warning("Enabling outlines cache. This is an unbounded on-disk "
45 - "cache. It may consume a lot of disk space and should "
46 - "not be used with untrusted clients.")
47 -else:
48 - disable_cache()
33 +CACHE = None
34
35
36 class BaseLogitsProcessor:
37
53 - def __init__(self, guide: Guide, reasoner: Optional[ReasoningParser]):
38 + def __init__(self, guide: Guide, eos_token_id: int,
39 + reasoner: Optional[ReasoningParser]) -> None:
40 self._guide: Guide = guide
41 + self._eos_token_id: int = eos_token_id
42 self._reasoner: Optional[ReasoningParser] = reasoner
56 - # CFGState is used for the FSM state for CFGGuide
57 - self._fsm_state: defaultdict[int, Union[int,
58 - CFGState]] = defaultdict(int)
59 -
60 - def clone(self) -> "BaseLogitsProcessor":
61 - cloned = copy.copy(self)
62 - cloned._guide = self._guide.copy()
63 - cloned._fsm_state = copy.deepcopy(self._fsm_state)
64 - return cloned
43 + self._mask: Optional[torch.Tensor] = None
44
45 def __call__(self, input_ids: list[int],
46 scores: torch.Tensor) -> torch.Tensor:
68 - """Use the FSM to bias the logits before sampling the next token."""
47 + if self._mask is None:
48 + self._mask = allocate_token_bitmask(scores.size(-1))
49
50 # Skip the structured logits processing if reasoning is not finished.
51 # reasoner is not None only when `--reasoning-parser` is set.
72 - if self._reasoner is not None:
73 - if not self._reasoner.is_reasoning_end(input_ids):
74 - return scores
75 - else:
76 - # Remove the reasoning tokens from the input_ids
77 - # We need this because our implementation relies on the
78 - # hash of the input_ids to store the FSM state.
79 - input_ids = self._reasoner.extract_content_ids(input_ids)
80 -
81 - seq_id = hash(tuple(input_ids))
82 -
83 - if len(input_ids) > 0:
84 - last_token = input_ids[-1]
85 - last_seq_id = hash(tuple(input_ids[:-1]))
86 - self._fsm_state[seq_id] = self._guide.get_next_state(
87 - state=self._fsm_state[last_seq_id], token_id=last_token)
88 - else:
89 - # Note: this is a hack.
90 - # Lark pickling does not work properly (silent failure),
91 - # which breaks the RPC (which uses python pickleing).
92 - # We need to find a better solution.
93 - # On the first time this is called, we simply re-create
94 - # the Lark object.
95 - if isinstance(self._guide, CFGGuide):
96 - self._guide.parser = PartialLark(
97 - self._guide.cfg_string,
98 - parser="lalr",
99 - import_paths=[grammars.GRAMMAR_PATH],
100 - )
101 - self._fsm_state[seq_id] = CFGState(
102 - parser_state=self._guide.parser.parse(""), prev_token=None)
103 -
104 - instruction = self._guide.get_next_instruction(
105 - state=self._fsm_state[seq_id])
106 -
107 - if type(instruction) == Generate: # noqa: E721
108 - allowed_tokens = instruction.tokens
109 - elif type(instruction) == Write: # noqa: E721
110 - # TODO: support fast forward tokens
111 - allowed_tokens = [instruction.tokens[0]]
112 - else:
113 - raise TypeError(
114 - f"Unsupported instruction type {type(instruction)}")
115 -
116 - mask = torch.full((scores.shape[-1], ),
117 - -torch.inf,
118 - device=scores.device)
119 - # The tokenizer may support more token ids than the model can generate,
120 - # eg. Llama 3.2 Vision models have an `<|image|>` token with id 128256
121 - # but scores.shape == torch.Size([128256])
122 - # Using NumPy is faster for filtering token ids
123 - allowed_tokens = np.array(allowed_tokens, dtype=np.int64)
124 - allowed_tokens = torch.tensor(allowed_tokens, device=scores.device)
125 - allowed_tokens = allowed_tokens.masked_select(
126 - allowed_tokens < scores.shape[-1])
127 - mask.index_fill_(0, allowed_tokens, 0)
128 - if current_platform.is_hpu():
129 - # Workaround for HPU bug where add_() raise RuntimeError:
130 - # synNodeCreateWithId failed for node: strided_insert
131 - # with synStatus 1 [Invalid argument], hopefully it will
132 - # be fixed in the future releases of the HPU runtime.
133 - scores = scores.add(mask)
134 - else:
135 - scores.add_(mask)
52 + if self._reasoner is not None and not self._reasoner.is_reasoning_end(
53 + input_ids):
54 + return scores
55 +
56 + # Remove the reasoning tokens from the input_ids
57 + # We need this because our implementation relies on the
58 + # input_ids sequence to store the FSM state.
59 + input_ids = (self._reasoner.extract_content_ids(input_ids)
60 + if self._reasoner is not None else input_ids)
61 +
62 + # Vllm V0 engine has a weird bug where we have to repeat
63 + # the eos token id twice for generation to stop, or at least
64 + # that is what we have to do from here in any case.
65 + # This is a patch until a better solution can be pushed
66 + # to outlines_core
67 + if input_ids and input_ids[-1] != self._eos_token_id:
68 + self._guide.advance(token_id=input_ids[-1], return_tokens=False)
69 +
70 + self._guide.write_mask_into(
71 + data_ptr=self._mask.data_ptr(),
72 + numel=self._mask.numel(),
73 + element_size=self._mask.element_size(),
74 + )
75 +
76 + # Any allowed tokens beyond the length of the scores will
77 + # be ignored by the kernel, taking care of the issue with
78 + # models such as Llama 3.2 Vision with an `<|image|>` token
79 + # with id 128256, but scores.shape == torch.Size([128256])
80 + _apply_token_bitmask_inplace_kernel(
81 + logits=scores.unsqueeze(dim=0),
82 + # mask must be on same device
83 + mask=self._mask.to(scores.device, non_blocking=True))
84 + self._mask.to("cpu", non_blocking=True)
85 +
86 return scores
87
88 + def clone(self) -> BaseLogitsProcessor:
89 + guide = copy.deepcopy(self._guide)
90 + guide.reset()
91 + return BaseLogitsProcessor(guide=guide,
92 + eos_token_id=self._eos_token_id,
93 + reasoner=self._reasoner)
94 +
95
96 class RegexLogitsProcessor(BaseLogitsProcessor):
97
98 @classmethod
142 - @cache()
99 def _get_guide(cls, regex_string: str,
100 tokenizer: PreTrainedTokenizerBase) -> Guide:
145 - tokenizer = _adapt_tokenizer(tokenizer)
146 - return RegexGuide.from_regex(regex_string, tokenizer)
147 -
148 - def __init__(
149 - self,
150 - regex_string: str,
151 - tokenizer: PreTrainedTokenizerBase,
152 - reasoner: Optional[ReasoningParser],
153 - ):
154 - """Compile the FSM that drives the regex-structured generation.
155 -
156 - Parameters
157 - ----------
158 - regex_string
159 - A string that represents a regular expression
160 - tokenizer
161 - The model's tokenizer
162 -
163 - """
101 + global CACHE
102 + if CACHE is None:
103 + CACHE = get_cache()
104 + vocabulary = get_vocabulary(tokenizer) # type: ignore[arg-type]
105 + cache_key = f"{vocabulary._hash}_{regex_string}"
106 + if CACHE is not None and cache_key in CACHE:
107 + return Guide(CACHE[cache_key])
108 +
109 + index = Index(regex_string, vocabulary.inner)
110 +
111 + if CACHE is not None:
112 + CACHE[cache_key] = index
113 +
114 + return Guide(index)
115 +
116 + def __init__(self, regex_string: str, tokenizer: PreTrainedTokenizerBase,
117 + reasoner: Optional[ReasoningParser]) -> None:
118 super().__init__(
165 - RegexLogitsProcessor._get_guide(regex_string, tokenizer), reasoner)
119 + guide=RegexLogitsProcessor._get_guide(regex_string, tokenizer),
120 + eos_token_id=tokenizer.eos_token_id, # type: ignore
121 + reasoner=reasoner)
122
123
124 class JSONLogitsProcessor(RegexLogitsProcessor):
@@ -170,22 +126,8 @@ class JSONLogitsProcessor(RegexLogitsProcessor):
126 def __init__(self, schema: Union[str, dict, BaseModel],
127 tokenizer: PreTrainedTokenizerBase,
128 whitespace_pattern: Union[str, None],
173 - reasoner: Optional[ReasoningParser]):
174 - """Compile the FSM that drives the JSON-guided generation.
175 -
176 - Parameters
177 - ----------
178 - schema
179 - A JSON schema that encodes the structure we want the model to
180 - generate
181 - tokenizer
182 - The model's tokenizer
183 - whitespace_pattern
184 - Pattern to use for JSON syntactic whitespace (doesn't impact
185 - string literals)
186 - Example: allow only a single space or newline with
187 - `whitespace_pattern=r"[\n ]?"`
188 - """
129 + reasoner: Optional[ReasoningParser]) -> None:
130 +
131 if isinstance(schema, type(BaseModel)):
132 schema_str = json.dumps(schema.model_json_schema())
133 elif isinstance(schema, dict):
@@ -197,63 +139,42 @@ class JSONLogitsProcessor(RegexLogitsProcessor):
139 f"Cannot parse schema {schema}. The schema must be either "
140 f"a Pydantic object, a dictionary or a string that contains "
141 f"the JSON Schema specification")
142 +
143 regex_string = build_regex_from_schema(schema_str, whitespace_pattern)
144 super().__init__(regex_string, tokenizer, reasoner)
145
146
204 -class CFGLogitsProcessor(BaseLogitsProcessor):
147 +class OutlinesVocabulary:
148 + """
149 + Wrapper class for `outlines_core.Vocabulary`,
150 + which allows us to store a hash with the vocabulary
151 + """
152 +
153 + def __init__(self, vocabulary: Vocabulary) -> None:
154 + # Actual vocabulary object
155 + self.inner = vocabulary
156 + # Have to do abs(hash()) because python hashes can
157 + # be negative, and we are using hash as a cache key.
158 + hex_str = hashlib.sha256(
159 + vocabulary.__repr__().encode('utf-8')).hexdigest()
160 + hash_int = int(hex_str, 16)
161 + self._hash = hash_int
162
206 - @classmethod
207 - @cache()
208 - def _get_guide(cls, cfg: str, tokenizer: PreTrainedTokenizerBase) -> Guide:
209 - tokenizer = _adapt_tokenizer(tokenizer)
210 - return CFGGuide(cfg, tokenizer)
211 -
212 - def __init__(self, cfg: str, tokenizer: PreTrainedTokenizerBase,
213 - reasoner: Optional[ReasoningParser]):
214 - """Compile the FSM that drives the context free grammar generation.
215 -
216 - Parameters
217 - ----------
218 - cfg
219 - A string that represents a context-free grammar
220 - tokenizer
221 - The model's tokenizer
222 -
223 - """
224 - super().__init__(CFGLogitsProcessor._get_guide(cfg, tokenizer),
225 - reasoner)
226 - self._guide = self._guide.copy()
227 -
228 - def clone(self) -> "CFGLogitsProcessor":
229 - cloned = copy.copy(self)
230 - cloned._fsm_state = copy.deepcopy(self._fsm_state)
231 - cloned._guide = self._guide.copy()
232 - return cloned
233 -
234 -
235 -@lru_cache(maxsize=32)
236 -def _adapt_tokenizer(tokenizer: PreTrainedTokenizerBase):
237 - """Adapt vLLM's tokenizer to use to compile the FSM.
238 -
239 - The API of Outlines tokenizers is slightly different to that of
240 - `transformers`. The decoder of outlines, returns a list whereas
241 - the decode of vLLM returns an str. To sync the vLLM decoder with
242 - outlines internal api, the decoder should be adapted. In addition
243 - we need to handle the missing spaces to Llama's tokenizer to be
244 - able to compile FSMs for this model.
163
246 - """
247 - if getattr(tokenizer, "_outlines_adapted", False):
248 - return tokenizer
164 +re_llama_byte_token = re.compile(r"^<0x[0-9A-F]{2}>$")
165 +re_replacement_seq = re.compile(r"^.{0,6}�+.{0,6}$")
166
250 - tokenizer = copy.deepcopy(tokenizer)
167
252 - tokenizer.vocabulary = tokenizer.get_vocab()
253 - tokenizer.special_tokens = set(tokenizer.all_special_tokens)
168 +def _reduced_vocabulary(tokenizer: AnyTokenizer,
169 + eos_token_id: int) -> dict[bytes, list[int]]:
170 + """Create a map from vocabulary tokens to lists of equivalent token ids.
171 +
172 + Returns:
173 + A Dict of token string -> equivalent token ids
174 + """
175 + unicode_to_bytes = {v: k for k, v in bytes_to_unicode().items()}
176
177 def convert_token_to_string(token: str) -> str:
256 - from transformers.file_utils import SPIECE_UNDERLINE
178
179 string = tokenizer.convert_tokens_to_string([token])
180
@@ -264,21 +185,123 @@ def _adapt_tokenizer(tokenizer: PreTrainedTokenizerBase):
185
186 return string
187
267 - def change_decoder(
268 - decoder: Callable[[list[int]],
269 - str]) -> Callable[[list[int]], list[str]]:
270 - """Sync vLLM's decoder with the outlines by returning list."""
188 + vocabulary: dict[bytes, list[int]] = {}
189 + empty_token_ids: list[int] = []
190 + for token, token_idx in tokenizer.get_vocab().items():
191 + if token in tokenizer.all_special_tokens: # type: ignore
192 + continue
193 +
194 + token_str = convert_token_to_string(token)
195 + if token_str:
196 + if isinstance(token, (bytes, bytearray)):
197 + # For BPE tokenizers where tokens are stored as bytes.
198 +
199 + # safe to ignore since token_str is of type (bytearray, bytes)
200 + # by this point.
201 + token_bytes = bytes(token_str) # type: ignore[arg-type]
202 +
203 + elif "\ufffd" in token_str and not re_replacement_seq.match(
204 + token_str):
205 + # Handle tokens with invalid UTF-8 sequences.
206 + if re_llama_byte_token.match(token):
207 + # Llama-like tokenizers use <0xXX> for incomplete sequences.
208 + token_bytes = bytes([int(token[3:5], 16)])
209 + else:
210 + # GPT2 tokenizers: map each byte back using unicode_to_bytes
211 + byte_vals = [unicode_to_bytes.get(c) for c in token]
212 + if None in byte_vals:
213 + raise RuntimeError(
214 + f"Cannot convert token `{token}`"
215 + f" ({token_idx}) to bytes: {token_str}")
216 + # safe to ignore, since if None in byte_vals,
217 + # an error is thrown.
218 + token_bytes = bytes(byte_vals) # type: ignore[arg-type]
219 + else:
220 + token_bytes = token_str.encode('utf-8')
221
272 - def new_decoder(inp_tokens: list[int]) -> list[str]:
273 - if (isinstance(inp_tokens, list) and len(inp_tokens) == 1
274 - and isinstance(inp_tokens[0], list)):
275 - inp_tokens = inp_tokens[0]
276 - return [decoder(inp_tokens)]
222 + if token_idx != eos_token_id:
223 + vocabulary.setdefault(token_bytes, []).append(token_idx)
224 + else:
225 + empty_token_ids.append(token_idx)
226
278 - return new_decoder
227 + return vocabulary
228
280 - tokenizer.convert_token_to_string = convert_token_to_string
281 - tokenizer.decode = change_decoder(tokenizer.decode)
282 - setattr(tokenizer, "_outlines_adapted", True) # noqa: B010
229
284 - return tokenizer
230 +def get_vocabulary(tokenizer: AnyTokenizer) -> Vocabulary:
231 + """Get the `Vocabulary` object for a given tokenizer.
232 + """
233 + if hasattr(tokenizer, "_outlines_vocabulary"):
234 + return tokenizer._outlines_vocabulary # type: ignore
235 +
236 + try:
237 + if hasattr(
238 + tokenizer,
239 + "eos_token_id",
240 + ) and tokenizer.eos_token_id is not None:
241 + eos_token_id = tokenizer.eos_token_id
242 + else:
243 + raise ValueError(
244 + f"Error during guided decoding setup: Tokenizer"
245 + f" ({type(tokenizer)}) has no `eos_token_id` property, "
246 + "but `eos_token_id` is required for guided decoding"
247 + " to work properly.")
248 +
249 + reduced_vocab = _reduced_vocabulary(
250 + tokenizer,
251 + eos_token_id #type: ignore
252 + )
253 + vocabulary = OutlinesVocabulary(Vocabulary(eos_token_id,
254 + reduced_vocab))
255 + tokenizer._outlines_vocabulary = vocabulary # type: ignore
256 +
257 + return vocabulary
258 + except AttributeError as e:
259 + raise ValueError(f"Cannot get the vocabulary of the tokenizer "
260 + f"({type(tokenizer)}). The tokenizer should have a "
261 + "get_vocab method.") from e
262 +
263 +
264 +def get_cache_path() -> str:
265 + """Get the context object that contains previously-computed return values"""
266 + outlines_cache_dir = os.getenv("OUTLINES_CACHE_DIR")
267 + xdg_cache_home = os.getenv("XDG_CACHE_HOME")
268 + home_dir = os.path.expanduser("~")
269 +
270 + if outlines_cache_dir:
271 + # OUTLINES_CACHE_DIR takes precedence
272 + return outlines_cache_dir
273 + elif xdg_cache_home:
274 + return os.path.join(xdg_cache_home, ".cache", "outlines")
275 + # If homedir is "/", we may be inside a container, and thus writing to
276 + # root would be problematic, so we fallback to using a tempfile.
277 + # Also validate the path exists, since os.path.expanduser does
278 + # not garuntee existence.
279 + elif os.path.isdir(home_dir) and home_dir != "/":
280 + # Default Unix fallback: ~/.cache/outlines
281 + return os.path.join(home_dir, ".cache", "outlines")
282 + else:
283 + import tempfile
284 +
285 + # home_dir may be / inside a docker container without existing user
286 + tempdir = tempfile.gettempdir()
287 + return os.path.join(tempdir, ".cache", "outlines")
288 +
289 +
290 +def get_cache():
291 + """Get the Cache instance to be used for index caching"""
292 +
293 + cache_dir = get_cache_path()
294 + if envs.VLLM_V0_USE_OUTLINES_CACHE:
295 + logger.warning("Enabling outlines cache. This is an unbounded on-disk "
296 + "cache. It may consume a lot of disk space and should "
297 + "not be used with untrusted clients.")
298 + cache = Cache(cache_dir, eviction_policy="none", cull_limit=0)
299 + outlines_version = importlib.metadata.version("outlines_core")
300 +
301 + cached_version = cache.get('__version__', None)
302 + if cached_version != outlines_version:
303 + cache.clear()
304 + cache.set('__version__', outlines_version)
305 + return cache
306 + else:
307 + return LRUCache(maxsize=128)
vllm/v1/engine/processor.py
+5
@@ -23,6 +23,8 @@ from vllm.v1.engine import EngineCoreRequest
23 from vllm.v1.engine.mm_input_cache import MirroredProcessingCache
24 from vllm.v1.structured_output.backend_guidance import (
25 validate_guidance_grammar)
26 +from vllm.v1.structured_output.backend_outlines import (
27 + validate_structured_output_request_outlines)
28 from vllm.v1.structured_output.backend_xgrammar import (
29 validate_xgrammar_grammar)
30
@@ -193,6 +195,9 @@ class Processor:
195 # https://github.com/guidance-ai/llguidance/blob/main/docs/syntax.md#special-tokens
196 # Without tokenizer these are disallowed in grammars.
197 validate_guidance_grammar(params, tokenizer=None)
198 + elif engine_level_backend == "outlines":
199 + # outlines backend
200 + validate_structured_output_request_outlines(params)
201 else:
202 # NOTE: engine_level_backend must be "auto" here, because we have
203 # checked supported_backends above.
vllm/v1/structured_output/__init__.py
+9
@@ -88,6 +88,15 @@ class StructuredOutputManager:
88 tokenizer=self.tokenizer,
89 vocab_size=vocab_size,
90 )
91 + elif backend == "outlines":
92 + from vllm.v1.structured_output.backend_outlines import (
93 + OutlinesBackend)
94 +
95 + self.backend = OutlinesBackend(
96 + self.vllm_config,
97 + tokenizer=self.tokenizer,
98 + vocab_size=vocab_size,
99 + )
100 else:
101 raise ValueError(
102 f"Unsupported structured output backend: {backend}")
vllm/v1/structured_output/backend_outlines.py new
+319
@@ -0,0 +1,319 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright 2025-present the Outlines developers
3 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
4 +from __future__ import annotations
5 +
6 +import ast
7 +import importlib
8 +import json
9 +import sys
10 +from dataclasses import dataclass, field
11 +from typing import TYPE_CHECKING
12 +
13 +import torch
14 +from regex import escape as regex_escape
15 +
16 +from vllm.model_executor.guided_decoding.outlines_logits_processors import (
17 + OutlinesVocabulary, get_cache, get_vocabulary)
18 +from vllm.sampling_params import SamplingParams
19 +from vllm.utils import LazyLoader
20 +from vllm.v1.structured_output.backend_types import (StructuredOutputBackend,
21 + StructuredOutputGrammar,
22 + StructuredOutputOptions)
23 +
24 +if TYPE_CHECKING:
25 + import outlines_core as oc
26 + import outlines_core.json_schema as json_schema
27 +else:
28 + oc = LazyLoader("oc", globals(), "outlines_core")
29 + json_schema = LazyLoader("json_schema", globals(),
30 + "outlines_core.json_schema")
31 +
32 +# Python 3.11+ sre_parse and sre_constants
33 +# are deprecated, so we must import them from re
34 +if sys.version_info >= (3, 11):
35 + # Hack to get around pre-commit regex module rule
36 + # because going through re is the only way to get sre_parse
37 + # and sre_constants in Python 3.11+
38 + _re = importlib.import_module("re")
39 + sre_parse = _re._parser
40 + sre_constants = _re._constants
41 +else:
42 + import sre_constants
43 + import sre_parse
44 +
45 +
46 +@dataclass
47 +class OutlinesBackend(StructuredOutputBackend):
48 +
49 + def __post_init__(self):
50 + self.vocabulary = get_vocabulary(self.tokenizer)
51 + self.cache = get_cache()
52 +
53 + def _compile_index(self, regex_string: str,
54 + vocabulary: OutlinesVocabulary) -> oc.Index:
55 + cache_key = f"{vocabulary._hash}_{regex_string}"
56 + if cache_key in self.cache:
57 + return self.cache[cache_key]
58 +
59 + index = oc.Index(regex_string, vocabulary.inner)
60 + self.cache[cache_key] = index
61 +
62 + return index
63 +
64 + def compile_grammar(self, request_type: StructuredOutputOptions,
65 + grammar_spec: str) -> StructuredOutputGrammar:
66 + if request_type == StructuredOutputOptions.JSON:
67 + regex = json_schema.build_regex_from_schema(grammar_spec)
68 + elif request_type == StructuredOutputOptions.REGEX:
69 + regex = grammar_spec
70 + elif request_type == StructuredOutputOptions.CHOICE:
71 + choices = ast.literal_eval(grammar_spec)
72 + choices = [regex_escape(c) for c in choices]
73 + regex = "(" + "|".join(choices) + ")"
74 + else:
75 + raise ValueError(
76 + f"Invalid request type for Outlines backend ({request_type!s})"
77 + )
78 + index = self._compile_index(regex, self.vocabulary)
79 + max_rollback_tokens = (
80 + self.vllm_config.speculative_config.num_speculative_tokens
81 + if self.vllm_config.speculative_config is not None else 0)
82 + return OutlinesGrammar(vocab_size=self.vocab_size,
83 + guide=oc.Guide(
84 + index, max_rollback=max_rollback_tokens))
85 +
86 + def allocate_token_bitmask(self, max_num_seqs: int) -> torch.Tensor:
87 + return torch.full(
88 + (max_num_seqs, (self.vocab_size + 31) // 32),
89 + -1,
90 + dtype=torch.int32,
91 + pin_memory=torch.cuda.is_available(),
92 + )
93 +
94 + def destroy(self):
95 + pass
96 +
97 +
98 +@dataclass
99 +class OutlinesGrammar(StructuredOutputGrammar):
100 +
101 + vocab_size: int
102 + guide: oc.Guide = field(hash=False)
103 + num_processed_tokens: int = field(default_factory=lambda: 0,
104 + repr=False,
105 + hash=False,
106 + init=False)
107 +
108 + # outlines_core signals done on DFA accept; vLLM expects done after EOS.
109 + # We delay the finished flag by one step so EOS can still be emitted.
110 + _prev_finished: bool = field(default=False,
111 + init=False,
112 + repr=False,
113 + hash=False)
114 +
115 + def accept_tokens(self, request_id: str, tokens: list[int]) -> bool:
116 + """Accepts a list of tokens and advances the FSM.
117 +
118 + Returns True if the FSM was advanced successfully.
119 + Returns False if the FSM failed to advance.
120 + """
121 + if self.guide.accepts_tokens(tokens):
122 + # Advance cannot fail because we checked Guide.accepts_tokens()
123 + for t in tokens:
124 + self.guide.advance(t)
125 + self.num_processed_tokens += 1
126 + return True
127 + return False
128 +
129 + def rollback(self, num_tokens: int) -> None:
130 + self.guide.rollback_state(num_tokens)
131 + self.num_processed_tokens -= num_tokens
132 +
133 + def validate_tokens(self, tokens: list[int]) -> list[int]:
134 + accepted: list[int] = []
135 + for tok in tokens:
136 + accepted.append(tok)
137 + if not self.guide.accepts_tokens(accepted):
138 + accepted.pop()
139 + break
140 + return accepted
141 +
142 + def fill_bitmask(self, bitmask: torch.Tensor, idx: int) -> None:
143 + mask = bitmask[idx]
144 + self.guide.write_mask_into(mask.data_ptr(), mask.numel(),
145 + mask.element_size())
146 +
147 + def is_terminated(self) -> bool:
148 + curr = self.guide.is_finished()
149 + prev = self._prev_finished
150 + self._prev_finished = curr
151 + return prev
152 +
153 + def reset(self):
154 + self.num_processed_tokens = 0
155 + self._prev_finished = False
156 + self.guide.reset()
157 +
158 +
159 +def validate_structured_output_request_outlines(params: SamplingParams):
160 + if params.guided_decoding is None:
161 + return
162 +
163 + gd_params = params.guided_decoding
164 +
165 + if gd_params.regex:
166 + validate_regex_is_buildable(gd_params.regex)
167 + elif gd_params.json:
168 + if isinstance(gd_params.json, str):
169 + try:
170 + # make sure schema is valid json
171 + json.loads(gd_params.json)
172 + schema = gd_params.json
173 + except json.JSONDecodeError as e:
174 + raise ValueError("Invalid JSON grammar specification.") from e
175 + else:
176 + try:
177 + schema = json.dumps(gd_params.json)
178 + except Exception as e:
179 + raise ValueError(
180 + f"Error serializing guided decoding jsonschema: {e}"
181 + ) from e
182 + pattern = json_schema.build_regex_from_schema(schema)
183 + validate_regex_is_buildable(pattern)
184 + elif gd_params.choice:
185 + choices = [regex_escape(str(choice)) for choice in gd_params.choice]
186 + regex = "(" + "|".join(choices) + ")"
187 + validate_regex_is_buildable(regex)
188 + elif gd_params.grammar:
189 + raise ValueError("Outlines guided decoding backend "
190 + "does not support grammar specifications")
191 +
192 +
193 +def _prefix_needs_context(parsed) -> bool:
194 + """Return True if there's a look-around/anchor before any consumer."""
195 +
196 + def subpattern_consumes(parsed) -> bool:
197 + """Return True if subpattern can consume at least one character."""
198 + tokens = parsed.data if hasattr(parsed, 'data') else parsed
199 + for ttype, tval in tokens:
200 + # literal, character class, or dot always consumes
201 + if ttype in (sre_parse.LITERAL, sre_parse.IN, sre_parse.ANY):
202 + return True
203 + # quantified subpattern: check inner pattern
204 + elif ttype == sre_parse.MAX_REPEAT:
205 + _, mx, sub = tval
206 + if mx != 0 and subpattern_consumes(sub):
207 + return True
208 + # alternation: if any branch consumes, the whole does
209 + elif ttype == sre_parse.BRANCH:
210 + _, branches = tval
211 + if any(subpattern_consumes(br) for br in branches):
212 + return True
213 + # grouped subpattern: recurse into its contents
214 + elif ttype == sre_parse.SUBPATTERN and subpattern_consumes(
215 + tval[3]):
216 + return True
217 + # No consumers, return False
218 + return False
219 +
220 + tokens = parsed.data if hasattr(parsed, 'data') else parsed
221 + for ttype, tval in tokens:
222 + # Direct anchors or look-around
223 + if ttype == sre_parse.AT or ttype in (sre_constants.ASSERT,
224 + sre_constants.ASSERT_NOT):
225 + return True
226 +
227 + # Nested subpattern: check
228 + if ttype == sre_parse.SUBPATTERN:
229 + # tval: (group, add_flags, del_flags, subpattern)
230 + if _prefix_needs_context(tval[3]):
231 + return True
232 + if subpattern_consumes(tval[3]):
233 + return False
234 +
235 + # if any branch has a prefix anchor => True,
236 + # else if at least one branch consumes => prefix ends => False
237 + elif ttype == sre_parse.BRANCH:
238 + saw_consumer = False
239 + for br in tval[1]:
240 + if _prefix_needs_context(br):
241 + return True
242 + if subpattern_consumes(br):
243 + saw_consumer = True
244 + if saw_consumer:
245 + return False
246 +
247 + # Immediate consumer tokens
248 + elif ttype in (sre_parse.LITERAL, sre_parse.IN, sre_parse.ANY):
249 + return False
250 +
251 + # if subpattern has anchor => True, if it can consume => stop
252 + elif ttype == sre_parse.MAX_REPEAT:
253 + if _prefix_needs_context(tval[2]):
254 + return True
255 + if subpattern_consumes(tval[2]):
256 + return False
257 +
258 + return False
259 +
260 +
261 +def _check_unsupported(parsed) -> None:
262 + """Check for regex features unsupported by regex-automata"""
263 + tokens = parsed.data if hasattr(parsed, 'data') else parsed
264 + for ttype, tval in tokens:
265 +
266 + # backreference
267 + if ttype in (sre_parse.GROUPREF, sre_parse.GROUPREF_EXISTS):
268 + raise ValueError("Backreferences are unsupported.")
269 +
270 + # look-around assertion
271 + elif ttype in (sre_constants.ASSERT, sre_constants.ASSERT_NOT):
272 + raise ValueError("Look-Around assertion are unsupported.")
273 +
274 + # unicode word boundaries
275 + elif ttype == sre_parse.AT:
276 + if tval in (sre_constants.AT_BOUNDARY,
277 + sre_constants.AT_NON_BOUNDARY):
278 + raise ValueError("Unicode word boundaries are unsupported.")
279 +
280 + elif ttype == sre_parse.BRANCH:
281 + # tval is (None, branches)
282 + for branch in tval[1]:
283 + _check_unsupported(branch)
284 +
285 + # tval is (min, max, subpattern)
286 + elif ttype == sre_parse.MAX_REPEAT:
287 + _check_unsupported(tval[2])
288 +
289 +
290 +def validate_regex_is_buildable(pattern: str) -> None:
291 + """
292 + Validates that the input regex is not using unsupported features
293 + of the `regex-automata` crate (outlines_core regex engine) and has a
294 + universal start state.
295 + definition of universal start state used can be found at:
296 + https://docs.rs/regex-automata/latest/regex_automata/dfa/trait.Automaton.html#method.universal_start_state
297 + """
298 + try:
299 + parsed = sre_parse.parse(pattern)
300 +
301 + except sre_constants.error as e:
302 + raise ValueError(f"Error parsing regex: {e}") from e
303 +
304 + try:
305 + _check_unsupported(parsed)
306 + except ValueError as e:
307 + raise ValueError(
308 + f"Regex uses unsupported feature for guided decoding: {e}. "
309 + "Only basic matching constructs are supported—lookarounds, "
310 + "backreferences, and unicode boundaries are not.") from e
311 +
312 + if _prefix_needs_context(parsed):
313 + raise ValueError(
314 + "Regex does not have a anchored universal start state"
315 + "This means that the Regex uses anchors (^) or look-arounds "
316 + "in a way which requires context before any token is matched."
317 + "Guided decoding needs regexes that can match without needing "
318 + "that context. Try rewriting the pattern without using these "
319 + f"constructs. Pattern:\n{pattern}")