A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 23:45 UTC
[V0 deprecation] Guided decoding (#21347)
Signed-off-by: Reza Barazesh <rezabarazesh@meta.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Reza Barazesh committed
Jul 29, 2025 at 03:15 UTC
37efc63b644b2f4e3b08bf7ff198dd8cd4c3f354
29 files changed
+103
-2809
.buildkite/test-pipeline.yaml
+1
-2
@@ -128,11 +128,10 @@ steps:
128
- tests/entrypoints/offline_mode
129
commands:
130
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
131
- - pytest -v -s entrypoints/llm --ignore=entrypoints/llm/test_lazy_outlines.py --ignore=entrypoints/llm/test_generate.py --ignore=entrypoints/llm/test_generate_multiple_loras.py --ignore=entrypoints/llm/test_guided_generate.py --ignore=entrypoints/llm/test_collective_rpc.py
131
+ - pytest -v -s entrypoints/llm --ignore=entrypoints/llm/test_lazy_outlines.py --ignore=entrypoints/llm/test_generate.py --ignore=entrypoints/llm/test_generate_multiple_loras.py --ignore=entrypoints/llm/test_collective_rpc.py
132
- pytest -v -s entrypoints/llm/test_lazy_outlines.py # it needs a clean process
133
- pytest -v -s entrypoints/llm/test_generate.py # it needs a clean process
134
- pytest -v -s entrypoints/llm/test_generate_multiple_loras.py # it needs a clean process
135
- - VLLM_USE_V1=0 pytest -v -s entrypoints/llm/test_guided_generate.py # it needs a clean process
135
- VLLM_USE_V1=0 pytest -v -s entrypoints/offline_mode # Needs to avoid interference with other tests
136
137
- label: Entrypoints Test (API Server) # 40min
.github/CODEOWNERS
-3
@@ -10,7 +10,6 @@
10
/vllm/worker/worker.py @zhuohan123 @youkaichao @alexm-redhat @comaniac @njhill
11
/vllm/model_executor/layers/sampler.py @zhuohan123 @youkaichao @alexm-redhat @comaniac @njhill
12
/vllm/model_executor/layers/quantization @mgoin @robertgshaw2-redhat @tlrmchlsmth
13
-/vllm/model_executor/guided_decoding @mgoin @russellb @aarnphm
13
/vllm/multimodal @DarkLight1337 @ywang96
14
/vllm/vllm_flash_attn @LucasWilkinson
15
/vllm/lora @jeejeelee
@@ -35,9 +34,7 @@ CMakeLists.txt @tlrmchlsmth @LucasWilkinson
34
/tests/distributed/test_pipeline_parallel.py @youkaichao
35
/tests/distributed/test_same_node.py @youkaichao
36
/tests/entrypoints @DarkLight1337 @robertgshaw2-redhat @simon-mo @aarnphm
38
-/tests/entrypoints/llm/test_guided_generate.py @mgoin @russellb @aarnphm
37
/tests/kernels @tlrmchlsmth @WoosukKwon
40
-/tests/model_executor/test_guided_processors.py @mgoin @russellb
38
/tests/models @DarkLight1337 @ywang96
39
/tests/multi_step @alexm-redhat @comaniac
40
/tests/multimodal @DarkLight1337 @ywang96
.github/mergify.yml
-3
@@ -149,9 +149,6 @@ pull_request_rules:
149
- files=examples/offline_inference/structured_outputs.py
150
- files=examples/online_serving/openai_chat_completion_structured_outputs.py
151
- files=examples/online_serving/openai_chat_completion_structured_outputs_with_reasoning.py
152
- - files~=^vllm/model_executor/guided_decoding/
153
- - files=tests/model_executor/test_guided_processors.py
154
- - files=tests/entrypoints/llm/test_guided_generate.py
152
- files~=^tests/v1/structured_output/
153
- files=tests/v1/entrypoints/llm/test_guided_generate.py
154
- files~=^vllm/v1/structured_output/
tests/entrypoints/llm/test_guided_generate.py
deleted
-552
@@ -1,552 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-import json
5
-import weakref
6
-from enum import Enum
7
-
8
-import jsonschema
9
-import pytest
10
-import regex as re
11
-from pydantic import BaseModel
12
-
13
-from vllm.distributed import cleanup_dist_env_and_memory
14
-from vllm.entrypoints.llm import LLM
15
-from vllm.outputs import RequestOutput
16
-from vllm.sampling_params import GuidedDecodingParams, SamplingParams
17
-
18
-MODEL_NAME = "Qwen/Qwen2.5-1.5B-Instruct"
19
-
20
-# Separate backends which support grammars vs ones
21
-# which only support regex based constraints in tests.
22
-GRAMMAR_DECODING_BACKENDS = [
23
- # (backend, disable_any_whitespace),
24
- ("lm-format-enforcer", False),
25
- ("xgrammar", True),
26
- ("guidance", True),
27
-]
28
-
29
-ALL_DECODING_BACKENDS = ([("outlines", False)] + GRAMMAR_DECODING_BACKENDS)
30
-
31
-
32
-@pytest.fixture(scope="module")
33
-def llm():
34
- # pytest caches the fixture so we use weakref.proxy to
35
- # enable garbage collection
36
- llm = LLM(model=MODEL_NAME, max_model_len=1024, seed=0)
37
-
38
- with llm.deprecate_legacy_api():
39
- yield weakref.proxy(llm)
40
- del llm
41
- cleanup_dist_env_and_memory()
42
-
43
-
44
-@pytest.mark.skip_global_cleanup
45
-@pytest.mark.parametrize("guided_decoding_backend,disable_any_whitespace",
46
- ALL_DECODING_BACKENDS)
47
-def test_guided_regex(sample_regex, llm, guided_decoding_backend: str,
48
- disable_any_whitespace: bool):
49
- sampling_params = SamplingParams(
50
- temperature=0.8,
51
- top_p=0.95,
52
- guided_decoding=GuidedDecodingParams(
53
- regex=sample_regex,
54
- backend=guided_decoding_backend,
55
- disable_any_whitespace=disable_any_whitespace))
56
-
57
- outputs = llm.generate(prompts=[
58
- f"Give an example IPv4 address with this regex: {sample_regex}"
59
- ] * 2,
60
- sampling_params=sampling_params,
61
- use_tqdm=True)
62
-
63
- assert outputs is not None
64
- for output in outputs:
65
- assert output is not None
66
- assert isinstance(output, RequestOutput)
67
- prompt = output.prompt
68
- generated_text = output.outputs[0].text
69
- print(generated_text)
70
- assert generated_text is not None
71
- assert re.fullmatch(sample_regex, generated_text) is not None
72
- print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
73
-
74
-
75
-@pytest.mark.skip_global_cleanup
76
-@pytest.mark.parametrize("guided_decoding_backend,disable_any_whitespace",
77
- ALL_DECODING_BACKENDS)
78
-def test_guided_json_completion(sample_json_schema, llm,
79
- guided_decoding_backend: str,
80
- disable_any_whitespace: bool):
81
- sampling_params = SamplingParams(
82
- temperature=1.0,
83
- max_tokens=1000,
84
- guided_decoding=GuidedDecodingParams(
85
- json=sample_json_schema,
86
- backend=guided_decoding_backend,
87
- disable_any_whitespace=disable_any_whitespace))
88
- outputs = llm.generate(prompts=[
89
- f"Give an example JSON for an employee profile "
90
- f"that fits this schema: {sample_json_schema}"
91
- ] * 2,
92
- sampling_params=sampling_params,
93
- use_tqdm=True)
94
-
95
- assert outputs is not None
96
-
97
- for output in outputs:
98
- assert output is not None
99
- assert isinstance(output, RequestOutput)
100
- prompt = output.prompt
101
-
102
- generated_text = output.outputs[0].text
103
- assert generated_text is not None
104
- print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
105
- output_json = json.loads(generated_text)
106
- jsonschema.validate(instance=output_json, schema=sample_json_schema)
107
-
108
-
109
-@pytest.mark.skip_global_cleanup
110
-@pytest.mark.parametrize("guided_decoding_backend,disable_any_whitespace",
111
- ALL_DECODING_BACKENDS)
112
-def test_guided_complex_json_completion(sample_complex_json_schema, llm,
113
- guided_decoding_backend: str,
114
- disable_any_whitespace: bool):
115
- sampling_params = SamplingParams(
116
- temperature=1.0,
117
- max_tokens=1000,
118
- guided_decoding=GuidedDecodingParams(
119
- json=sample_complex_json_schema,
120
- backend=guided_decoding_backend,
121
- disable_any_whitespace=disable_any_whitespace))
122
- outputs = llm.generate(prompts=[
123
- f"Give an example JSON for an assignment grade "
124
- f"that fits this schema: {sample_complex_json_schema}"
125
- ] * 2,
126
- sampling_params=sampling_params,
127
- use_tqdm=True)
128
-
129
- assert outputs is not None
130
-
131
- for output in outputs:
132
- assert output is not None
133
- assert isinstance(output, RequestOutput)
134
- prompt = output.prompt
135
-
136
- generated_text = output.outputs[0].text
137
- assert generated_text is not None
138
- print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
139
- output_json = json.loads(generated_text)
140
- jsonschema.validate(instance=output_json,
141
- schema=sample_complex_json_schema)
142
-
143
-
144
-@pytest.mark.skip_global_cleanup
145
-@pytest.mark.parametrize("guided_decoding_backend,disable_any_whitespace",
146
- ALL_DECODING_BACKENDS)
147
-def test_guided_definition_json_completion(sample_definition_json_schema, llm,
148
- guided_decoding_backend: str,
149
- disable_any_whitespace: bool):
150
- sampling_params = SamplingParams(
151
- temperature=1.0,
152
- max_tokens=1000,
153
- guided_decoding=GuidedDecodingParams(
154
- json=sample_definition_json_schema,
155
- backend=guided_decoding_backend,
156
- disable_any_whitespace=disable_any_whitespace))
157
- outputs = llm.generate(prompts=[
158
- f"Give an example JSON for solving 8x + 7 = -23 "
159
- f"that fits this schema: {sample_definition_json_schema}"
160
- ] * 2,
161
- sampling_params=sampling_params,
162
- use_tqdm=True)
163
-
164
- assert outputs is not None
165
-
166
- for output in outputs:
167
- assert output is not None
168
- assert isinstance(output, RequestOutput)
169
- prompt = output.prompt
170
-
171
- generated_text = output.outputs[0].text
172
- assert generated_text is not None
173
- print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
174
- output_json = json.loads(generated_text)
175
- jsonschema.validate(instance=output_json,
176
- schema=sample_definition_json_schema)
177
-
178
-
179
-@pytest.mark.skip_global_cleanup
180
-@pytest.mark.parametrize("guided_decoding_backend,disable_any_whitespace",
181
- ALL_DECODING_BACKENDS)
182
-def test_guided_enum_json_completion(sample_enum_json_schema, llm,
183
- guided_decoding_backend: str,
184
- disable_any_whitespace: bool):
185
- sampling_params = SamplingParams(
186
- temperature=1.0,
187
- max_tokens=1000,
188
- guided_decoding=GuidedDecodingParams(
189
- json=sample_enum_json_schema,
190
- backend=guided_decoding_backend,
191
- disable_any_whitespace=disable_any_whitespace))
192
- outputs = llm.generate(prompts=[
193
- "Create a bug report JSON that fits this schema: "
194
- f"{sample_enum_json_schema}. Make it for a high priority critical bug."
195
- ] * 2,
196
- sampling_params=sampling_params,
197
- use_tqdm=True)
198
-
199
- assert outputs is not None
200
-
201
- for output in outputs:
202
- assert output is not None
203
- assert isinstance(output, RequestOutput)
204
- prompt = output.prompt
205
-
206
- generated_text = output.outputs[0].text
207
- assert generated_text is not None
208
- print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
209
- output_json = json.loads(generated_text)
210
- jsonschema.validate(instance=output_json,
211
- schema=sample_enum_json_schema)
212
-
213
- # Additional assertions to verify enum values
214
- assert output_json["status"] in ["active", "inactive", "pending"]
215
- assert output_json["priority"] in ["low", "medium", "high", "critical"]
216
- assert output_json["category"]["type"] in [
217
- "bug", "feature", "improvement"
218
- ]
219
- assert output_json["category"]["severity"] in [1, 2, 3, 4, 5]
220
- for flag in output_json["flags"]:
221
- assert flag in ["urgent", "blocked", "needs_review", "approved"]
222
-
223
-
224
-@pytest.mark.skip_global_cleanup
225
-@pytest.mark.parametrize("guided_decoding_backend,disable_any_whitespace",
226
- ALL_DECODING_BACKENDS)
227
-def test_guided_choice_completion(sample_guided_choice, llm,
228
- guided_decoding_backend: str,
229
- disable_any_whitespace: bool):
230
- sampling_params = SamplingParams(
231
- temperature=0.8,
232
- top_p=0.95,
233
- guided_decoding=GuidedDecodingParams(
234
- choice=sample_guided_choice,
235
- backend=guided_decoding_backend,
236
- disable_any_whitespace=disable_any_whitespace))
237
- outputs = llm.generate(
238
- prompts="The best language for type-safe systems programming is ",
239
- sampling_params=sampling_params,
240
- use_tqdm=True)
241
-
242
- assert outputs is not None
243
- for output in outputs:
244
- assert output is not None
245
- assert isinstance(output, RequestOutput)
246
- prompt = output.prompt
247
- generated_text = output.outputs[0].text
248
- print(generated_text)
249
- assert generated_text is not None
250
- assert generated_text in sample_guided_choice
251
- print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
252
-
253
-
254
-@pytest.mark.skip_global_cleanup
255
-@pytest.mark.parametrize("guided_decoding_backend,disable_any_whitespace",
256
- GRAMMAR_DECODING_BACKENDS)
257
-def test_guided_grammar(sample_sql_statements, llm,
258
- guided_decoding_backend: str,
259
- disable_any_whitespace: bool):
260
- sampling_params = SamplingParams(
261
- temperature=0.8,
262
- top_p=0.95,
263
- max_tokens=1000,
264
- guided_decoding=GuidedDecodingParams(
265
- grammar=sample_sql_statements,
266
- backend=guided_decoding_backend,
267
- disable_any_whitespace=disable_any_whitespace))
268
- outputs = llm.generate(
269
- prompts=("Generate a sql state that select col_1 from "
270
- "table_1 where it is equals to 1"),
271
- sampling_params=sampling_params,
272
- use_tqdm=True,
273
- )
274
-
275
- assert outputs is not None
276
- for output in outputs:
277
- assert output is not None
278
- assert isinstance(output, RequestOutput)
279
- prompt = output.prompt
280
-
281
- generated_text = output.outputs[0].text
282
- assert generated_text is not None
283
- # use Lark to parse the output, and make sure it's a valid parse tree
284
- from lark import Lark
285
- parser = Lark(sample_sql_statements)
286
- parser.parse(generated_text)
287
-
288
- # remove spaces for comparison b/c we removed them in the grammar
289
- ground_truth = "SELECT col_1 from table_1 where col_1 = 1".replace(
290
- " ", "")
291
-
292
- assert generated_text.strip() == ground_truth
293
-
294
- print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
295
-
296
-
297
-@pytest.mark.skip_global_cleanup
298
-def test_guided_options_request_deprecation_warning(sample_regex, llm):
299
- sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
300
-
301
- with pytest.warns(DeprecationWarning, match="guided_options_request"):
302
- llm.generate(prompts="This should fail",
303
- sampling_params=sampling_params,
304
- use_tqdm=True,
305
- guided_options_request=dict(guided_regex=sample_regex))
306
-
307
-
308
-@pytest.mark.skip_global_cleanup
309
-def test_validation_against_both_guided_decoding_options(sample_regex, llm):
310
- sampling_params = SamplingParams(
311
- temperature=0.8,
312
- top_p=0.95,
313
- guided_decoding=GuidedDecodingParams(regex=sample_regex))
314
-
315
- with pytest.raises(ValueError, match="Cannot set both"):
316
- llm.generate(prompts="This should fail",
317
- sampling_params=sampling_params,
318
- use_tqdm=True,
319
- guided_options_request=dict(guided_regex=sample_regex))
320
-
321
-
322
-@pytest.mark.skip_global_cleanup
323
-def test_disable_guided_decoding_fallback(sample_regex, llm):
324
- # see has_xgrammar_unsupported_json_features()
325
- unsupported_json = {
326
- "type": "object",
327
- "properties": {
328
- "example": {
329
- "type": "string",
330
- "minLength": 5 # unsupported by xgrammar
331
- }
332
- }
333
- }
334
- sampling_params = SamplingParams(temperature=0.8,
335
- top_p=0.95,
336
- guided_decoding=GuidedDecodingParams(
337
- json=unsupported_json,
338
- backend="xgrammar",
339
- disable_fallback=True))
340
-
341
- with pytest.raises(
342
- ValueError,
343
- match="xgrammar does not support advanced JSON schema features "
344
- "like string length, item limits, or property bounds."):
345
- llm.generate(prompts="This should fail",
346
- sampling_params=sampling_params,
347
- use_tqdm=True)
348
-
349
-
350
-@pytest.mark.skip_global_cleanup
351
-@pytest.mark.parametrize("guided_decoding_backend,disable_any_whitespace",
352
- GRAMMAR_DECODING_BACKENDS)
353
-def test_guided_json_object(llm, guided_decoding_backend: str,
354
- disable_any_whitespace: bool):
355
- sampling_params = SamplingParams(
356
- temperature=1.0,
357
- max_tokens=100,
358
- n=2,
359
- guided_decoding=GuidedDecodingParams(
360
- json_object=True,
361
- backend=guided_decoding_backend,
362
- disable_any_whitespace=disable_any_whitespace))
363
-
364
- outputs = llm.generate(
365
- prompts=("Generate a JSON object with curly braces for a person with "
366
- "name and age fields for John Smith who is 31 years old."),
367
- sampling_params=sampling_params,
368
- use_tqdm=True)
369
-
370
- assert outputs is not None
371
- for output in outputs:
372
- assert output is not None
373
- assert isinstance(output, RequestOutput)
374
-
375
- for i in range(2):
376
- generated_text = output.outputs[i].text
377
- print(generated_text)
378
- assert generated_text is not None
379
-
380
- if disable_any_whitespace:
381
- assert "\n" not in generated_text
382
-
383
- # Parse to verify it is valid JSON
384
- parsed_json = json.loads(generated_text)
385
- # A list is not what was intended, but is still valid
386
- # json.
387
- assert isinstance(parsed_json, (dict, list))
388
-
389
-
390
-class CarType(str, Enum):
391
- sedan = "sedan"
392
- suv = "SUV"
393
- truck = "Truck"
394
- coupe = "Coupe"
395
-
396
-
397
-class CarDescription(BaseModel):
398
- brand: str
399
- model: str
400
- car_type: CarType
401
-
402
-
403
-@pytest.mark.skip_global_cleanup
404
-@pytest.mark.parametrize("guided_decoding_backend,disable_any_whitespace",
405
- ALL_DECODING_BACKENDS)
406
-def test_guided_json_completion_with_enum(llm, guided_decoding_backend: str,
407
- disable_any_whitespace: bool):
408
- json_schema = CarDescription.model_json_schema()
409
- sampling_params = SamplingParams(
410
- temperature=1.0,
411
- max_tokens=1000,
412
- guided_decoding=GuidedDecodingParams(
413
- json=json_schema,
414
- backend=guided_decoding_backend,
415
- disable_any_whitespace=disable_any_whitespace))
416
- outputs = llm.generate(
417
- prompts="Generate a JSON with the brand, model and car_type of"
418
- "the most iconic car from the 90's",
419
- sampling_params=sampling_params,
420
- use_tqdm=True)
421
-
422
- assert outputs is not None
423
- for output in outputs:
424
- assert output is not None
425
- assert isinstance(output, RequestOutput)
426
- prompt = output.prompt
427
-
428
- generated_text = output.outputs[0].text
429
- assert generated_text is not None
430
- print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
431
- output_json = json.loads(generated_text)
432
- jsonschema.validate(instance=output_json, schema=json_schema)
433
-
434
-
435
-@pytest.mark.skip_global_cleanup
436
-@pytest.mark.parametrize("guided_decoding_backend,disable_any_whitespace",
437
- ALL_DECODING_BACKENDS)
438
-def test_guided_number_range_json_completion(llm, guided_decoding_backend: str,
439
- disable_any_whitespace: bool):
440
- sample_output_schema = {
441
- "type": "object",
442
- "properties": {
443
- "age": {
444
- "type": "integer",
445
- "minimum": 18,
446
- "maximum": 99
447
- },
448
- "score": {
449
- "type": "number",
450
- "minimum": 0.0,
451
- "maximum": 100.0
452
- },
453
- "zipcode": {
454
- "type": "string",
455
- "pattern": r"^\d{5}(-\d{4})?$"
456
- },
457
- },
458
- "required": ["age", "score", "zipcode"],
459
- }
460
- sampling_params = SamplingParams(
461
- temperature=1.0,
462
- max_tokens=1000,
463
- guided_decoding=GuidedDecodingParams(
464
- json=sample_output_schema,
465
- backend=guided_decoding_backend,
466
- disable_any_whitespace=disable_any_whitespace),
467
- )
468
- outputs = llm.generate(
469
- prompts=[
470
- "Create a JSON object for a user with age, score, and zipcode."
471
- ] * 2,
472
- sampling_params=sampling_params,
473
- use_tqdm=True,
474
- )
475
-
476
- assert outputs is not None
477
-
478
- for output in outputs:
479
- assert output is not None
480
- assert isinstance(output, RequestOutput)
481
- prompt = output.prompt
482
-
483
- generated_text = output.outputs[0].text
484
- assert generated_text is not None
485
- print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
486
- output_json = json.loads(generated_text)
487
- jsonschema.validate(instance=output_json, schema=sample_output_schema)
488
- assert 18 <= output_json["age"] <= 99
489
- assert 0.0 <= output_json["score"] <= 100.0
490
- assert (re.fullmatch(r"^\d{5}(-\d{4})?$", output_json["zipcode"])
491
- is not None)
492
-
493
-
494
-@pytest.mark.skip_global_cleanup
495
-def test_guidance_no_additional_properties(llm):
496
- schema = {
497
- 'type': 'object',
498
- 'properties': {
499
- 'a1': {
500
- 'type': 'string'
501
- },
502
- 'a2': {
503
- 'type': 'string'
504
- },
505
- 'a3': {
506
- 'type': 'string'
507
- }
508
- },
509
- 'required': ['a1', 'a2', 'a3'],
510
- }
511
-
512
- prompt = (
513
- "<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a "
514
- "helpful assistant.<|im_end|>\n<|im_start|>user\nPlease generate a "
515
- "large JSON object with key-value pairs a1=b1, a2=b2, ..., a20=b20"
516
- "<|im_end|>\n<|im_start|>assistant\n")
517
-
518
- def generate_with_backend(backend, disable_additional_properties):
519
- guided_params = GuidedDecodingParams(
520
- json=schema,
521
- backend=backend,
522
- disable_any_whitespace=True,
523
- disable_additional_properties=disable_additional_properties)
524
- sampling_params = SamplingParams(temperature=0,
525
- max_tokens=256,
526
- guided_decoding=guided_params)
527
-
528
- outputs = llm.generate(prompts=prompt, sampling_params=sampling_params)
529
- assert outputs is not None
530
- generated_text = outputs[0].outputs[0].text
531
- assert generated_text is not None
532
- parsed_json = json.loads(generated_text)
533
- assert isinstance(parsed_json, dict)
534
- jsonschema.validate(instance=parsed_json, schema=schema)
535
- return parsed_json
536
-
537
- base_generated = generate_with_backend("guidance", False)
538
- assert "a1" in base_generated
539
- assert "a2" in base_generated
540
- assert "a3" in base_generated
541
- # by default additional keys are generated
542
- assert "a4" in base_generated
543
- assert "a5" in base_generated
544
- assert "a6" in base_generated
545
-
546
- generated = generate_with_backend("guidance", True)
547
- assert "a1" in generated
548
- assert "a2" in generated
549
- assert "a3" in generated
550
- assert "a4" not in generated
551
- assert "a5" not in generated
552
- assert "a6" not in generated
tests/entrypoints/llm/test_lazy_outlines.py
+11
-41
@@ -4,43 +4,11 @@
4
import sys
5
from contextlib import nullcontext
6
7
-import pytest
7
from vllm_test_utils import BlameResult, blame
8
9
from vllm import LLM, SamplingParams
10
from vllm.distributed import cleanup_dist_env_and_memory
12
-
13
-
14
-@pytest.fixture(scope="function", autouse=True)
15
-def use_v0_only(monkeypatch):
16
- """
17
- V1 only supports xgrammar so this is irrelevant.
18
- """
19
- monkeypatch.setenv('VLLM_USE_V1', '0')
20
-
21
-
22
-def run_normal_opt125m():
23
- prompts = [
24
- "Hello, my name is",
25
- "The president of the United States is",
26
- "The capital of France is",
27
- "The future of AI is",
28
- ]
29
- sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
30
-
31
- # Create an LLM without guided decoding as a baseline.
32
- llm = LLM(model="facebook/opt-125m",
33
- enforce_eager=True,
34
- gpu_memory_utilization=0.3)
35
- outputs = llm.generate(prompts, sampling_params)
36
- for output in outputs:
37
- prompt = output.prompt
38
- generated_text = output.outputs[0].text
39
- print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
40
-
41
- # Destroy the LLM object and free up the GPU memory.
42
- del llm
43
- cleanup_dist_env_and_memory()
11
+from vllm.sampling_params import GuidedDecodingParams
12
13
14
def run_normal():
@@ -67,20 +35,22 @@ def run_normal():
35
cleanup_dist_env_and_memory()
36
37
70
-def run_lmfe(sample_regex):
38
+def run_xgrammar(sample_regex):
39
# Create an LLM with guided decoding enabled.
40
llm = LLM(model="distilbert/distilgpt2",
41
enforce_eager=True,
74
- guided_decoding_backend="lm-format-enforcer",
42
+ guided_decoding_backend="xgrammar",
43
gpu_memory_utilization=0.3)
76
- sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
44
+ prompt = f"Give an example IPv4 address with this regex: {sample_regex}"
45
+ guided_decoding = GuidedDecodingParams(regex=sample_regex)
46
+ sampling_params = SamplingParams(temperature=0.8,
47
+ top_p=0.95,
48
+ guided_decoding=guided_decoding)
49
outputs = llm.generate(
78
- prompts=[
79
- f"Give an example IPv4 address with this regex: {sample_regex}"
80
- ] * 2,
50
+ prompts=[prompt] * 2,
51
sampling_params=sampling_params,
52
use_tqdm=True,
83
- guided_options_request=dict(guided_regex=sample_regex))
53
+ )
54
55
for output in outputs:
56
prompt = output.prompt
@@ -103,7 +73,7 @@ def test_lazy_outlines(sample_regex):
73
lambda: module_name in sys.modules) if use_blame else nullcontext()
74
with context as result:
75
run_normal()
106
- run_lmfe(sample_regex)
76
+ run_xgrammar(sample_regex)
77
if use_blame:
78
assert isinstance(result, BlameResult)
79
print(f"the first import location is:\n{result.trace_stack}")
tests/entrypoints/openai/test_chat.py
+20
-131
@@ -488,7 +488,9 @@ async def test_chat_completion_stream_options(client: openai.AsyncOpenAI,
488
489
@pytest.mark.asyncio
490
async def test_guided_choice_chat(client: openai.AsyncOpenAI,
491
- sample_guided_choice):
491
+ sample_guided_choice, is_v1_server: bool):
492
+ if not is_v1_server:
493
+ pytest.skip("Guided decoding is only supported in v1 engine")
494
messages = [{
495
"role": "system",
496
"content": "you are a helpful assistant"
@@ -524,8 +526,10 @@ async def test_guided_choice_chat(client: openai.AsyncOpenAI,
526
527
528
@pytest.mark.asyncio
527
-async def test_guided_json_chat(client: openai.AsyncOpenAI,
528
- sample_json_schema):
529
+async def test_guided_json_chat(client: openai.AsyncOpenAI, sample_json_schema,
530
+ is_v1_server: bool):
531
+ if not is_v1_server:
532
+ pytest.skip("Guided decoding is only supported in v1 engine")
533
534
messages = [{
535
"role": "system",
@@ -568,7 +572,10 @@ async def test_guided_json_chat(client: openai.AsyncOpenAI,
572
573
574
@pytest.mark.asyncio
571
-async def test_guided_regex_chat(client: openai.AsyncOpenAI, sample_regex):
575
+async def test_guided_regex_chat(client: openai.AsyncOpenAI, sample_regex,
576
+ is_v1_server: bool):
577
+ if not is_v1_server:
578
+ pytest.skip("Guided decoding is only supported in v1 engine")
579
580
messages = [{
581
"role": "system",
@@ -653,7 +660,10 @@ async def test_guided_choice_chat_logprobs(client: openai.AsyncOpenAI,
660
661
662
@pytest.mark.asyncio
656
-async def test_named_tool_use(client: openai.AsyncOpenAI, sample_json_schema):
663
+async def test_named_tool_use(client: openai.AsyncOpenAI, sample_json_schema,
664
+ is_v1_server: bool):
665
+ if not is_v1_server:
666
+ pytest.skip("Tool use is only supported in v1 engine")
667
messages = [{
668
"role": "system",
669
"content": "you are a helpful assistant"
@@ -741,131 +751,6 @@ async def test_named_tool_use(client: openai.AsyncOpenAI, sample_json_schema):
751
assert json1["age"] != json2["age"]
752
753
744
-@pytest.mark.asyncio
745
-@pytest.mark.parametrize("model_name", [MODEL_NAME])
746
-async def test_required_tool_use(client: openai.AsyncOpenAI,
747
- is_v1_server: bool, model_name: str):
748
- if is_v1_server:
749
- pytest.skip(
750
- "tool_choice='required' requires features unsupported on V1")
751
-
752
- tools = [
753
- {
754
- "type": "function",
755
- "function": {
756
- "name": "get_current_weather",
757
- "description": "Get the current weather in a given location",
758
- "parameters": {
759
- "type": "object",
760
- "properties": {
761
- "city": {
762
- "type": "string",
763
- "description":
764
- "The city to find the weather for, e.g. 'Vienna'",
765
- "default": "Vienna",
766
- },
767
- "country": {
768
- "type":
769
- "string",
770
- "description":
771
- "The country that the city is in, e.g. 'Austria'",
772
- },
773
- "unit": {
774
- "type": "string",
775
- "description":
776
- "The unit to fetch the temperature in",
777
- "enum": ["celsius", "fahrenheit"],
778
- },
779
- },
780
- "required": ["country", "unit"],
781
- },
782
- },
783
- },
784
- {
785
- "type": "function",
786
- "function": {
787
- "name": "get_forecast",
788
- "description": "Get the weather forecast for a given location",
789
- "parameters": {
790
- "type": "object",
791
- "properties": {
792
- "city": {
793
- "type": "string",
794
- "description":
795
- "The city to get the forecast for, e.g. 'Vienna'",
796
- "default": "Vienna",
797
- },
798
- "country": {
799
- "type":
800
- "string",
801
- "description":
802
- "The country that the city is in, e.g. 'Austria'",
803
- },
804
- "days": {
805
- "type":
806
- "integer",
807
- "description":
808
- "Number of days to get the forecast for (1-7)",
809
- },
810
- "unit": {
811
- "type": "string",
812
- "description":
813
- "The unit to fetch the temperature in",
814
- "enum": ["celsius", "fahrenheit"],
815
- },
816
- },
817
- "required": ["country", "days", "unit"],
818
- },
819
- },
820
- },
821
- ]
822
-
823
- messages = [
824
- {
825
- "role": "user",
826
- "content": "Hi! How are you doing today?"
827
- },
828
- {
829
- "role": "assistant",
830
- "content": "I'm doing well! How can I help you?"
831
- },
832
- {
833
- "role":
834
- "user",
835
- "content":
836
- "Can you tell me what the current weather is in Berlin and the "\
837
- "forecast for the next 5 days, in fahrenheit?",
838
- },
839
- ]
840
-
841
- # Non-streaming test
842
- chat_completion = await client.chat.completions.create(
843
- messages=messages,
844
- model=model_name,
845
- tools=tools,
846
- tool_choice="required",
847
- )
848
-
849
- assert chat_completion.choices[0].message.tool_calls is not None
850
- assert len(chat_completion.choices[0].message.tool_calls) > 0
851
-
852
- # Streaming test
853
- stream = await client.chat.completions.create(
854
- messages=messages,
855
- model=model_name,
856
- tools=tools,
857
- tool_choice="required",
858
- stream=True,
859
- )
860
-
861
- output = []
862
- async for chunk in stream:
863
- if chunk.choices and chunk.choices[0].delta.tool_calls:
864
- output.extend(chunk.choices[0].delta.tool_calls)
865
-
866
- assert len(output) > 0
867
-
868
-
754
@pytest.mark.asyncio
755
async def test_inconsistent_tool_choice_and_tools(client: openai.AsyncOpenAI,
756
sample_json_schema):
@@ -948,7 +833,11 @@ async def test_response_format_json_object(client: openai.AsyncOpenAI):
833
834
835
@pytest.mark.asyncio
951
-async def test_response_format_json_schema(client: openai.AsyncOpenAI):
836
+async def test_response_format_json_schema(client: openai.AsyncOpenAI,
837
+ is_v1_server: bool):
838
+ if not is_v1_server:
839
+ pytest.skip(
840
+ "JSON schema response format is only supported in v1 engine")
841
prompt = 'what is 1+1? The format is "result": 2'
842
# Check that this prompt cannot lead to a valid JSON without json_schema
843
for _ in range(2):
tests/entrypoints/openai/test_completion.py
+30
-6
@@ -28,7 +28,7 @@ MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
28
# but we're not testing generation quality here
29
LORA_NAME = "typeof/zephyr-7b-beta-lora"
30
31
-GUIDED_DECODING_BACKENDS = ["outlines", "lm-format-enforcer", "xgrammar"]
31
+GUIDED_DECODING_BACKENDS = ["outlines", "xgrammar", "guidance"]
32
33
34
@pytest.fixture(scope="module")
@@ -95,6 +95,14 @@ def server(default_server_args, request):
95
os.environ['VLLM_USE_V1'] = original_value
96
97
98
+@pytest.fixture
99
+def is_v1_server(server):
100
+ import os
101
+
102
+ # For completion tests, we assume v0 since there's no explicit v1 setup
103
+ return os.environ.get('VLLM_USE_V1', '0') == '1'
104
+
105
+
106
@pytest_asyncio.fixture
107
async def client(server):
108
async with server.get_async_client() as async_client:
@@ -631,7 +639,10 @@ async def test_allowed_token_ids(client: openai.AsyncOpenAI):
639
@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)
640
async def test_guided_json_completion(client: openai.AsyncOpenAI,
641
guided_decoding_backend: str,
634
- sample_json_schema):
642
+ sample_json_schema, is_v1_server: bool):
643
+ if not is_v1_server:
644
+ pytest.skip("Guided decoding is only supported in v1 engine")
645
+
646
completion = await client.completions.create(
647
model=MODEL_NAME,
648
prompt=f"Give an example JSON for an employee profile "
@@ -653,7 +664,10 @@ async def test_guided_json_completion(client: openai.AsyncOpenAI,
664
@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)
665
async def test_guided_regex_completion(client: openai.AsyncOpenAI,
666
guided_decoding_backend: str,
656
- sample_regex):
667
+ sample_regex, is_v1_server: bool):
668
+ if not is_v1_server:
669
+ pytest.skip("Guided decoding is only supported in v1 engine")
670
+
671
completion = await client.completions.create(
672
model=MODEL_NAME,
673
prompt=f"Give an example IPv4 address with this regex: {sample_regex}",
@@ -674,7 +688,11 @@ async def test_guided_regex_completion(client: openai.AsyncOpenAI,
688
@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)
689
async def test_guided_choice_completion(client: openai.AsyncOpenAI,
690
guided_decoding_backend: str,
677
- sample_guided_choice):
691
+ sample_guided_choice,
692
+ is_v1_server: bool):
693
+ if not is_v1_server:
694
+ pytest.skip("Guided decoding is only supported in v1 engine")
695
+
696
completion = await client.completions.create(
697
model=MODEL_NAME,
698
prompt="The best language for type-safe systems programming is ",
@@ -692,7 +710,9 @@ async def test_guided_choice_completion(client: openai.AsyncOpenAI,
710
711
@pytest.mark.asyncio
712
async def test_guided_grammar(client: openai.AsyncOpenAI,
695
- sample_sql_statements):
713
+ sample_sql_statements, is_v1_server: bool):
714
+ if not is_v1_server:
715
+ pytest.skip("Guided grammar is only supported in v1 engine")
716
717
completion = await client.completions.create(
718
model=MODEL_NAME,
@@ -754,7 +774,11 @@ async def test_echo_logprob_completion(client: openai.AsyncOpenAI,
774
@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)
775
async def test_guided_decoding_type_error(client: openai.AsyncOpenAI,
776
guided_decoding_backend: str,
757
- sample_json_schema, sample_regex):
777
+ sample_json_schema, sample_regex,
778
+ is_v1_server: bool):
779
+ if not is_v1_server:
780
+ pytest.skip("Guided decoding is only supported in v1 engine")
781
+
782
with pytest.raises(openai.BadRequestError):
783
_ = await client.completions.create(
784
model=MODEL_NAME,
tests/entrypoints/openai/test_prompt_validation.py
+5
-21
@@ -9,6 +9,11 @@ import regex as re
9
from ...utils import RemoteOpenAIServer
10
11
12
+@pytest.fixture(scope="function", autouse=True)
13
+def use_v1_only(monkeypatch):
14
+ monkeypatch.setenv('VLLM_USE_V1', '1')
15
+
16
+
17
@pytest.mark.asyncio
18
async def test_empty_prompt():
19
model_name = "gpt2"
@@ -37,24 +42,3 @@ async def test_out_of_vocab_token_ids():
42
prompt=[999999],
43
max_tokens=5,
44
temperature=0.0)
40
-
41
-
42
-@pytest.mark.asyncio
43
-async def test_reject_multistep_with_guided_decoding():
44
- model_name = "gpt2"
45
- server_args = ["--enforce-eager", "--num-scheduler-steps", "8"]
46
- with RemoteOpenAIServer(model_name, server_args) as remote_server:
47
- client = remote_server.get_async_client()
48
-
49
- with pytest.raises(
50
- openai.BadRequestError,
51
- match=re.compile(
52
- '.*Guided decoding .* multi-step decoding.*').pattern):
53
- await client.completions.create(
54
- model=model_name,
55
- prompt="Hello",
56
- max_tokens=5,
57
- temperature=0.0,
58
- extra_body={"response_format": {
59
- "type": "json_object"
60
- }})
tests/model_executor/test_guided_processors.py
deleted
-207
@@ -1,207 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-import json
5
-import pickle
6
-
7
-import pytest
8
-import torch
9
-from transformers import AutoTokenizer
10
-
11
-from vllm.config import ModelConfig
12
-from vllm.model_executor.guided_decoding import (
13
- get_guided_decoding_logits_processor,
14
- get_local_guided_decoding_logits_processor)
15
-from vllm.model_executor.guided_decoding.outlines_logits_processors import (
16
- JSONLogitsProcessor, RegexLogitsProcessor)
17
-from vllm.sampling_params import GuidedDecodingParams
18
-
19
-MODEL_NAME = 'HuggingFaceH4/zephyr-7b-beta'
20
-GUIDED_DECODING_BACKENDS = [
21
- "outlines", "lm-format-enforcer", "xgrammar", "guidance"
22
-]
23
-GUIDED_DECODING_BACKENDS_WITH_REASONING_SUPPORT = ["outlines", "xgrammar"]
24
-REASONING_MODEL_NAME = "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B"
25
-
26
-
27
-# Initialize the tokenizer for the model here to avoid repeated loading
28
-@pytest.fixture(scope="module")
29
-def zephyr_7B_tokenzer():
30
- return AutoTokenizer.from_pretrained(MODEL_NAME)
31
-
32
-
33
-@pytest.fixture(scope="module")
34
-def deepseek_r1_qwen_tokenizer():
35
- return AutoTokenizer.from_pretrained(REASONING_MODEL_NAME)
36
-
37
-
38
-def test_guided_logits_processors(zephyr_7B_tokenzer, sample_regex,
39
- sample_json_schema):
40
- """Basic unit test for RegexLogitsProcessor and JSONLogitsProcessor."""
41
- regex_LP = RegexLogitsProcessor(sample_regex,
42
- zephyr_7B_tokenzer,
43
- reasoner=None)
44
- json_LP = JSONLogitsProcessor(sample_json_schema,
45
- zephyr_7B_tokenzer,
46
- whitespace_pattern=None,
47
- reasoner=None)
48
-
49
- tensor = torch.rand(32000)
50
- original_tensor = torch.clone(tensor)
51
- tensor = regex_LP([], tensor)
52
- assert tensor.shape == original_tensor.shape
53
- assert not torch.allclose(tensor, original_tensor)
54
-
55
- tensor = torch.rand(32000)
56
- original_tensor = torch.clone(tensor)
57
- tensor = json_LP([], tensor)
58
- assert tensor.shape == original_tensor.shape
59
- assert not torch.allclose(tensor, original_tensor)
60
-
61
-
62
-@pytest.mark.asyncio
63
-@pytest.mark.parametrize("backend", GUIDED_DECODING_BACKENDS)
64
-@pytest.mark.parametrize("is_local", [True, False])
65
-async def test_guided_logits_processor_black_box(backend: str, is_local: bool,
66
- sample_regex,
67
- sample_json_schema,
68
- zephyr_7B_tokenzer):
69
-
70
- config = ModelConfig(
71
- MODEL_NAME,
72
- runner="generate",
73
- seed=0,
74
- dtype="bfloat16",
75
- )
76
- regex_request = GuidedDecodingParams(regex=sample_regex, backend=backend)
77
-
78
- regex_lp = get_local_guided_decoding_logits_processor(
79
- regex_request, zephyr_7B_tokenzer, config) if is_local else \
80
- await get_guided_decoding_logits_processor(
81
- regex_request, zephyr_7B_tokenzer, config)
82
- assert regex_lp is not None
83
- tensor = torch.rand(32000)
84
- original_tensor = torch.clone(tensor)
85
- # allowed tokens at state 0
86
- tensor = regex_lp([], tensor)
87
- assert tensor.shape == original_tensor.shape
88
- assert not torch.allclose(tensor, original_tensor)
89
-
90
- json_request = GuidedDecodingParams(json=sample_json_schema,
91
- backend=backend)
92
- json_lp = await get_guided_decoding_logits_processor(
93
- json_request, zephyr_7B_tokenzer, config)
94
- assert json_lp is not None
95
- tensor = torch.rand(32000)
96
- original_tensor = torch.clone(tensor)
97
- tensor = json_lp([], tensor)
98
- assert tensor.shape == original_tensor.shape
99
- assert not torch.allclose(tensor, original_tensor)
100
-
101
-
102
-@pytest.mark.asyncio
103
-@pytest.mark.parametrize("backend",
104
- GUIDED_DECODING_BACKENDS_WITH_REASONING_SUPPORT)
105
-@pytest.mark.parametrize("is_local", [True, False])
106
-@pytest.mark.parametrize("reasoning_backend", ["deepseek_r1"])
107
-async def test_guided_logits_processor_with_reasoning(
108
- backend: str, is_local: bool, reasoning_backend: str, sample_regex,
109
- sample_json_schema, deepseek_r1_qwen_tokenizer):
110
-
111
- config = ModelConfig(
112
- REASONING_MODEL_NAME,
113
- runner="generate",
114
- seed=0,
115
- dtype="bfloat16",
116
- )
117
- token_ids = deepseek_r1_qwen_tokenizer.encode(
118
- "<think>here is the thinking process")
119
- regex_request = GuidedDecodingParams(regex=sample_regex, backend=backend)
120
-
121
- regex_lp = get_local_guided_decoding_logits_processor(regex_request,
122
- deepseek_r1_qwen_tokenizer, config,
123
- reasoning_backend) if is_local else \
124
- await get_guided_decoding_logits_processor(
125
- regex_request, deepseek_r1_qwen_tokenizer, config,
126
- reasoning_backend)
127
- assert regex_lp is not None
128
- tensor = torch.rand(151664)
129
- original_tensor = torch.clone(tensor)
130
- tensor = regex_lp(token_ids, tensor)
131
- assert tensor.shape == original_tensor.shape
132
- assert torch.allclose(tensor, original_tensor)
133
-
134
- token_ids = deepseek_r1_qwen_tokenizer.encode(
135
- "<think>here is the thinking process")
136
- json_request = GuidedDecodingParams(json=sample_json_schema,
137
- backend=backend)
138
- json_lp = get_local_guided_decoding_logits_processor(
139
- json_request, deepseek_r1_qwen_tokenizer, config,
140
- reasoning_backend) if is_local else \
141
- await get_guided_decoding_logits_processor(
142
- json_request, deepseek_r1_qwen_tokenizer, config, reasoning_backend)
143
- assert json_lp is not None
144
- tensor = torch.rand(151664)
145
- original_tensor = torch.clone(tensor)
146
- tensor = json_lp(token_ids, tensor)
147
- assert tensor.shape == original_tensor.shape
148
- assert torch.allclose(tensor, original_tensor)
149
-
150
- # Thinking is over, so the tensor should change.
151
- token_ids = deepseek_r1_qwen_tokenizer.encode(
152
- "<think>here is the thinking process</think>")
153
- json_request = GuidedDecodingParams(json=sample_json_schema,
154
- backend=backend)
155
- json_lp = get_local_guided_decoding_logits_processor(
156
- json_request, deepseek_r1_qwen_tokenizer, config,
157
- reasoning_backend) if is_local else \
158
- await get_guided_decoding_logits_processor(
159
- json_request, deepseek_r1_qwen_tokenizer, config, reasoning_backend)
160
- assert json_lp is not None
161
- tensor = torch.rand(151664)
162
- original_tensor = torch.clone(tensor)
163
- tensor = json_lp(token_ids, tensor)
164
- assert tensor.shape == original_tensor.shape
165
- assert not torch.allclose(tensor, original_tensor)
166
-
167
-
168
-def test_multiple_guided_options_not_allowed(sample_json_schema, sample_regex):
169
- with pytest.raises(ValueError,
170
- match="You can only use one kind of guided"):
171
- GuidedDecodingParams(json=sample_json_schema, regex=sample_regex)
172
-
173
- with pytest.raises(ValueError,
174
- match="You can only use one kind of guided"):
175
- GuidedDecodingParams(json=sample_json_schema, json_object=True)
176
-
177
- with pytest.raises(ValueError,
178
- match="You can only use one kind of guided"):
179
- GuidedDecodingParams(json=sample_json_schema, choice=["a", "b"])
180
-
181
- with pytest.raises(ValueError,
182
- match="You can only use one kind of guided"):
183
- GuidedDecodingParams(json=sample_json_schema, grammar="test grammar")
184
-
185
-
186
-def test_pickle_xgrammar_tokenizer_data():
187
- try:
188
- import xgrammar as xgr
189
- except ImportError:
190
- pytest.skip("Could not import xgrammar to run test")
191
-
192
- from vllm.model_executor.guided_decoding.xgrammar_decoding import (
193
- TokenizerData)
194
- tokenizer_data = TokenizerData(
195
- metadata=
196
- '{"vocab_type":2,"vocab_size":151665,"add_prefix_space":false,"stop_token_ids":[151645]}',
197
- encoded_vocab=['!', '"', '#', '$', '%'],
198
- )
199
- pickled = pickle.dumps(tokenizer_data)
200
-
201
- assert pickled is not None
202
-
203
- depickled: TokenizerData = pickle.loads(pickled)
204
-
205
- assert depickled is not None
206
- assert json.loads(
207
- depickled.metadata)['vocab_type'] == xgr.VocabType.BYTE_LEVEL.value
tests/models/language/generation/test_mistral.py
+1
-50
@@ -3,13 +3,11 @@
3
import copy
4
import json
5
6
-import jsonschema
7
-import jsonschema.exceptions
6
import pytest
7
8
from vllm.entrypoints.openai.tool_parsers.mistral_tool_parser import (
9
MistralToolCall, MistralToolParser)
12
-from vllm.sampling_params import GuidedDecodingParams, SamplingParams
10
+from vllm.sampling_params import SamplingParams
11
from vllm.transformers_utils.tokenizer import MistralTokenizer
12
13
from ...utils import check_logprobs_close
@@ -274,53 +272,6 @@ def test_mistral_function_calling(vllm_runner, model: str, dtype: str) -> None:
272
assert parsed_message.content is None
273
274
277
-@pytest.mark.parametrize("model", MODELS)
278
-@pytest.mark.parametrize("guided_backend",
279
- ["outlines", "lm-format-enforcer", "xgrammar"])
280
-def test_mistral_guided_decoding(
281
- monkeypatch: pytest.MonkeyPatch,
282
- vllm_runner,
283
- model: str,
284
- guided_backend: str,
285
-) -> None:
286
- with monkeypatch.context() as m:
287
- # Guided JSON not supported in xgrammar + V1 yet
288
- m.setenv("VLLM_USE_V1", "0")
289
-
290
- with vllm_runner(
291
- model,
292
- dtype='bfloat16',
293
- tokenizer_mode="mistral",
294
- guided_decoding_backend=guided_backend,
295
- ) as vllm_model:
296
- guided_decoding = GuidedDecodingParams(json=SAMPLE_JSON_SCHEMA)
297
- params = SamplingParams(max_tokens=512,
298
- temperature=0.7,
299
- guided_decoding=guided_decoding)
300
-
301
- messages = [{
302
- "role": "system",
303
- "content": "you are a helpful assistant"
304
- }, {
305
- "role":
306
- "user",
307
- "content":
308
- f"Give an example JSON for an employee profile that "
309
- f"fits this schema: {SAMPLE_JSON_SCHEMA}"
310
- }]
311
- outputs = vllm_model.llm.chat(messages, sampling_params=params)
312
-
313
- generated_text = outputs[0].outputs[0].text
314
- json_response = json.loads(generated_text)
315
- assert outputs is not None
316
-
317
- try:
318
- jsonschema.validate(instance=json_response,
319
- schema=SAMPLE_JSON_SCHEMA)
320
- except jsonschema.exceptions.ValidationError:
321
- pytest.fail("Generated response is not valid with JSON schema")
322
-
323
-
275
def test_mistral_function_call_nested_json():
276
"""Ensure that the function-name regex captures the entire outer-most
277
JSON block, including nested braces."""
tests/samplers/test_no_bad_words.py
+3
-3
@@ -14,9 +14,9 @@ from vllm import LLM, SamplingParams
14
15
16
@pytest.fixture(autouse=True)
17
-def v1(run_with_both_engines):
18
- """We can run both engines for this test."""
19
- pass
17
+def v1(monkeypatch):
18
+ """Only run on vLLM v1."""
19
+ monkeypatch.setenv('VLLM_USE_V1', '1')
20
21
22
def _generate(
tests/test_sampling_params.py
+1
-2
@@ -56,8 +56,7 @@ def test_sampling_params_from_request_with_no_guided_decoding_backend(
56
57
58
@pytest.mark.parametrize("request_level_guided_decoding_backend,expected",
59
- [("xgrammar", "xgrammar"),
60
- ("lm-format-enforcer", "lm-format-enforcer"),
59
+ [("xgrammar", "xgrammar"), ("guidance", "guidance"),
60
("outlines", "outlines")])
61
def test_sampling_params_from_request_with_guided_decoding_backend(
62
request_level_guided_decoding_backend: str, expected: str,
tests/v1/test_oracle.py
-7
@@ -47,13 +47,6 @@ def test_unsupported_configs(monkeypatch):
47
},
48
).create_engine_config()
49
50
- with pytest.raises(NotImplementedError):
51
- AsyncEngineArgs(
52
- model=MODEL,
53
- guided_decoding_backend="lm-format-enforcer",
54
- guided_decoding_disable_fallback=True,
55
- ).create_engine_config()
56
-
50
with pytest.raises(NotImplementedError):
51
AsyncEngineArgs(
52
model=MODEL,
tools/check_pickle_imports.py
-1
@@ -34,7 +34,6 @@ ALLOWED_FILES = set([
34
'vllm/model_executor/models/registry.py',
35
'tests/test_utils.py',
36
'tests/tokenization/test_cached_tokenizer.py',
37
- 'tests/model_executor/test_guided_processors.py',
37
'vllm/distributed/utils.py',
38
'vllm/distributed/parallel_state.py',
39
'vllm/engine/multiprocessing/client.py',
vllm/config.py
+4
-16
@@ -1774,8 +1774,8 @@ class CacheConfig:
1774
- "builtin" is Python's built-in hash.\n
1775
- "sha256" is collision resistant but with certain overheads.
1776
This option uses Pickle for object serialization before hashing.\n
1777
- - "sha256_cbor_64bit" provides a reproducible, cross-language compatible
1778
- hash. It serializes objects using canonical CBOR and hashes them with
1777
+ - "sha256_cbor_64bit" provides a reproducible, cross-language compatible
1778
+ hash. It serializes objects using canonical CBOR and hashes them with
1779
SHA-256. The resulting hash consists of the lower 64 bits of the SHA-256
1780
digest."""
1781
cpu_offload_gb: float = 0
@@ -3721,12 +3721,7 @@ def get_served_model_name(model: str,
3721
return served_model_name
3722
3723
3724
-GuidedDecodingBackendV0 = Literal["auto", "outlines", "lm-format-enforcer",
3725
- "xgrammar", "guidance"]
3726
-
3727
-GuidedDecodingBackendV1 = Literal["auto", "xgrammar", "guidance", "outlines"]
3728
-GuidedDecodingBackend = Literal[GuidedDecodingBackendV0,
3729
- GuidedDecodingBackendV1]
3724
+GuidedDecodingBackend = Literal["auto", "xgrammar", "guidance", "outlines"]
3725
3726
3727
@config
@@ -3734,7 +3729,7 @@ GuidedDecodingBackend = Literal[GuidedDecodingBackendV0,
3729
class DecodingConfig:
3730
"""Dataclass which contains the decoding strategy of the engine."""
3731
3737
- backend: GuidedDecodingBackend = "auto" if envs.VLLM_USE_V1 else "xgrammar"
3732
+ backend: GuidedDecodingBackend = "auto"
3733
"""Which engine will be used for guided decoding (JSON schema / regex etc)
3734
by default. With "auto", we will make opinionated choices based on request
3735
contents and what the backend libraries currently support, so the behavior
@@ -3776,13 +3771,6 @@ class DecodingConfig:
3771
return hash_str
3772
3773
def __post_init__(self):
3779
- if envs.VLLM_USE_V1:
3780
- valid_guided_backends = get_args(GuidedDecodingBackendV1)
3781
- else:
3782
- valid_guided_backends = get_args(GuidedDecodingBackendV0)
3783
- if self.backend not in valid_guided_backends:
3784
- raise ValueError(f"Invalid backend '{self.backend}',"
3785
- f" must be one of {valid_guided_backends}")
3774
if (self.disable_any_whitespace
3775
and self.backend not in ("xgrammar", "guidance")):
3776
raise ValueError("disable_any_whitespace is only supported for "
vllm/engine/arg_utils.py
+8
-16
@@ -25,14 +25,14 @@ from vllm.config import (BlockSize, CacheConfig, CacheDType, CompilationConfig,
25
ConfigFormat, ConfigType, ConvertOption,
26
DecodingConfig, DetailedTraceModules, Device,
27
DeviceConfig, DistributedExecutorBackend,
28
- GuidedDecodingBackend, GuidedDecodingBackendV1,
29
- HfOverrides, KVEventsConfig, KVTransferConfig,
30
- LoadConfig, LogprobsMode, LoRAConfig, ModelConfig,
31
- ModelDType, ModelImpl, MultiModalConfig,
32
- ObservabilityConfig, ParallelConfig, PoolerConfig,
33
- PrefixCachingHashAlgo, RunnerOption, SchedulerConfig,
34
- SchedulerPolicy, SpeculativeConfig, TaskOption,
35
- TokenizerMode, VllmConfig, get_attr_docs, get_field)
28
+ GuidedDecodingBackend, HfOverrides, KVEventsConfig,
29
+ KVTransferConfig, LoadConfig, LogprobsMode,
30
+ LoRAConfig, ModelConfig, ModelDType, ModelImpl,
31
+ MultiModalConfig, ObservabilityConfig, ParallelConfig,
32
+ PoolerConfig, PrefixCachingHashAlgo, RunnerOption,
33
+ SchedulerConfig, SchedulerPolicy, SpeculativeConfig,
34
+ TaskOption, TokenizerMode, VllmConfig, get_attr_docs,
35
+ get_field)
36
from vllm.logger import init_logger
37
from vllm.platforms import CpuArchEnum, current_platform
38
from vllm.plugins import load_general_plugins
@@ -1343,14 +1343,6 @@ class EngineArgs:
1343
recommend_to_remove=True)
1344
return False
1345
1346
- if self.guided_decoding_backend not in get_args(
1347
- GuidedDecodingBackendV1):
1348
- _raise_or_fallback(
1349
- feature_name=
1350
- f"--guided-decoding-backend={self.guided_decoding_backend}",
1351
- recommend_to_remove=False)
1352
- return False
1353
-
1346
# Need at least Ampere for now (FA support required).
1347
# Skip this check if we are running on a non-GPU platform,
1348
# or if the device capability is not available
vllm/engine/async_llm_engine.py
+4
-62
@@ -2,7 +2,6 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
import asyncio
5
-import copy
5
import time
6
import weakref
7
from functools import partial
@@ -24,8 +23,6 @@ from vllm.inputs import PromptType
23
from vllm.inputs.preprocess import InputPreprocessor
24
from vllm.logger import init_logger
25
from vllm.lora.request import LoRARequest
27
-from vllm.model_executor.guided_decoding import (
28
- get_guided_decoding_logits_processor)
26
from vllm.model_executor.layers.sampler import SamplerOutput
27
from vllm.outputs import PoolingRequestOutput, RequestOutput
28
from vllm.pooling_params import PoolingParams
@@ -469,19 +466,6 @@ class _AsyncLLMEngine(LLMEngine):
466
tokenization_kwargs=tokenization_kwargs,
467
)
468
472
- if isinstance(params, SamplingParams) and \
473
- params.guided_decoding is not None:
474
- # Guided decoding has an async implementation for building logits
475
- # processors in a separate threadpool.
476
- # We want to invoke that here instead of using the blocking
477
- # implementation in the LLMEngine
478
- params = await build_guided_decoding_logits_processor_async(
479
- sampling_params=params,
480
- tokenizer=await self.get_tokenizer_async(lora_request),
481
- default_guided_backend=self.decoding_config.backend,
482
- reasoning_backend=self.decoding_config.reasoning_backend,
483
- model_config=self.model_config)
484
-
469
self._add_processed_request(
470
request_id=request_id,
471
processed_inputs=processed_inputs,
@@ -503,48 +487,6 @@ class _AsyncLLMEngine(LLMEngine):
487
raise NotImplementedError
488
489
506
-async def build_guided_decoding_logits_processor_async(
507
- sampling_params: SamplingParams, tokenizer: AnyTokenizer,
508
- default_guided_backend: str, reasoning_backend: Optional[str],
509
- model_config: ModelConfig) -> SamplingParams:
510
- """Constructs logits processors based on the guided_decoding,
511
- logits_bias, and allowed_token_ids fields in sampling_params. Deletes
512
- those fields and adds the constructed logits processors to the
513
- logits_processors field. Modifies sampling params in-place and returns
514
- the modified sampling params."""
515
- if sampling_params.guided_decoding is None:
516
- return sampling_params
517
-
518
- # Defensively copy sampling params since guided decoding logits
519
- # processors can have different state for each request
520
- sampling_params = copy.copy(sampling_params)
521
- guided_decoding = sampling_params.guided_decoding
522
-
523
- logger.debug(
524
- "Building guided decoding logits processor. "
525
- "guided_decoding: %s%s", guided_decoding,
526
- f", reasoning_backend: {reasoning_backend}"
527
- if reasoning_backend is not None else "")
528
-
529
- guided_decoding.backend = guided_decoding.backend or default_guided_backend
530
-
531
- processor = await get_guided_decoding_logits_processor(
532
- guided_params=guided_decoding,
533
- tokenizer=tokenizer,
534
- reasoning_backend=reasoning_backend,
535
- model_config=model_config)
536
-
537
- if processor:
538
- if sampling_params.logits_processors is None:
539
- sampling_params.logits_processors = []
540
- sampling_params.logits_processors.append(processor)
541
-
542
- # Unset guided decoding params after constructing the lp from them
543
- sampling_params.guided_decoding = None
544
-
545
- return sampling_params
546
-
547
-
490
class AsyncLLMEngine(EngineClient):
491
"""An asynchronous wrapper for [`LLMEngine`][vllm.LLMEngine].
492
@@ -1028,7 +970,7 @@ class AsyncLLMEngine(EngineClient):
970
```
971
# Please refer to entrypoints/api_server.py for
972
# the complete example.
1031
-
973
+
974
# initialize the engine and the example input
975
# note that engine_args here is AsyncEngineArgs instance
976
engine = AsyncLLMEngine.from_engine_args(engine_args)
@@ -1036,13 +978,13 @@ class AsyncLLMEngine(EngineClient):
978
"input": "What is LLM?",
979
"request_id": 0,
980
}
1039
-
981
+
982
# start the generation
983
results_generator = engine.encode(
984
example_input["input"],
985
PoolingParams(),
986
example_input["request_id"])
1045
-
987
+
988
# get the results
989
final_output = None
990
async for request_output in results_generator:
@@ -1052,7 +994,7 @@ class AsyncLLMEngine(EngineClient):
994
# Return or raise an error
995
...
996
final_output = request_output
1055
-
997
+
998
# Process and return the final output
999
...
1000
```
vllm/engine/llm_engine.py
+7
-41
@@ -1,7 +1,6 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
-import copy
4
import time
5
from collections import Counter as collectionsCounter
6
from collections import deque
@@ -36,8 +35,6 @@ from vllm.inputs.preprocess import InputPreprocessor
35
from vllm.logger import init_logger
36
from vllm.logits_process import get_bad_words_logits_processors
37
from vllm.lora.request import LoRARequest
39
-from vllm.model_executor.guided_decoding import (
40
- get_local_guided_decoding_logits_processor)
38
from vllm.model_executor.layers.sampler import SamplerOutput
39
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
40
from vllm.multimodal.processing import EncDecMultiModalProcessor
@@ -686,11 +683,10 @@ class LLMEngine:
683
"Priority scheduling is not enabled.")
684
685
if isinstance(params, SamplingParams) \
689
- and (params.guided_decoding or params.logits_processors) \
686
+ and params.logits_processors \
687
and self.scheduler_config.num_scheduler_steps > 1:
688
raise ValueError(
692
- "Guided decoding and logits processors are not supported "
693
- "in multi-step decoding")
689
+ "Logits processors are not supported in multi-step decoding")
690
691
if arrival_time is None:
692
arrival_time = time.time()
@@ -1226,7 +1222,7 @@ class LLMEngine:
1222
engine = LLMEngine.from_engine_args(engine_args)
1223
example_inputs = [(0, "What is LLM?",
1224
SamplingParams(temperature=0.0))]
1229
-
1225
+
1226
# Start the engine with an event loop
1227
while True:
1228
if example_inputs:
@@ -1983,43 +1979,13 @@ class LLMEngine:
1979
def _build_logits_processors(
1980
self, sampling_params: SamplingParams,
1981
lora_request: Optional[LoRARequest]) -> SamplingParams:
1986
- """Constructs logits processors based on the guided_decoding,
1987
- logits_bias, and allowed_token_ids fields in sampling_params. Deletes
1988
- those fields and adds the constructed logits processors to the
1989
- logits_processors field. Returns the modified sampling params."""
1982
+ """Constructs logits processors based on the logits_bias, and
1983
+ allowed_token_ids fields in sampling_params. Deletes those fields and
1984
+ adds the constructed logits processors to the logits_processors field.
1985
+ Returns the modified sampling params."""
1986
1987
logits_processors = []
1988
1993
- if sampling_params.guided_decoding is not None:
1994
- # Defensively copy sampling params since guided decoding logits
1995
- # processors can have different state for each request
1996
- sampling_params = copy.copy(sampling_params)
1997
- guided_decoding = sampling_params.guided_decoding
1998
-
1999
- logger.debug(
2000
- "Building guided decoding logits processor in "
2001
- "LLMEngine. Params: %s", guided_decoding)
2002
-
2003
- tokenizer = self.get_tokenizer(lora_request=lora_request)
2004
- guided_decoding.backend = guided_decoding.backend or \
2005
- self.decoding_config.backend
2006
-
2007
- if self.decoding_config.reasoning_backend:
2008
- logger.debug("Building with reasoning backend %s",
2009
- self.decoding_config.reasoning_backend)
2010
-
2011
- processor = get_local_guided_decoding_logits_processor(
2012
- guided_params=guided_decoding,
2013
- tokenizer=tokenizer,
2014
- model_config=self.model_config,
2015
- reasoning_backend=self.decoding_config.reasoning_backend,
2016
- )
2017
- if processor:
2018
- logits_processors.append(processor)
2019
-
2020
- # Unset so this doesn't get passed down to the model
2021
- sampling_params.guided_decoding = None
2022
-
1989
if (sampling_params.logit_bias or sampling_params.allowed_token_ids):
1990
tokenizer = self.get_tokenizer(lora_request=lora_request)
1991
vllm/engine/multiprocessing/client.py
-18
@@ -20,8 +20,6 @@ from vllm.config import DecodingConfig, ModelConfig, VllmConfig
20
from vllm.core.scheduler import SchedulerOutputs
21
# yapf conflicts with isort for this block
22
# yapf: disable
23
-from vllm.engine.async_llm_engine import (
24
- build_guided_decoding_logits_processor_async)
23
from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT,
24
IPC_HEALTH_EXT, IPC_INPUT_EXT,
25
IPC_OUTPUT_EXT, RPC_REQUEST_T,
@@ -537,22 +535,6 @@ class MQLLMEngineClient(EngineClient):
535
if request_id in self.output_queues:
536
raise ValueError(f"Request {request_id} already exists")
537
540
- # Constructing guided decoding logits processors is expensive, so we do
541
- # it here to avoid contending with cpu resources and the GIL on the
542
- # backend process.
543
- if isinstance(params, SamplingParams) and \
544
- params.guided_decoding is not None:
545
- params = await \
546
- build_guided_decoding_logits_processor_async(
547
- sampling_params=params,
548
- tokenizer=await self.get_tokenizer(lora_request),
549
- default_guided_backend=(self.decoding_config.backend
550
- if self.decoding_config
551
- else DecodingConfig.backend),
552
- model_config=self.model_config,
553
- reasoning_backend=self.decoding_config.reasoning_backend,
554
- )
555
-
538
# 1) Create output queue for this requests.
539
queue: asyncio.Queue[Union[RequestOutput,
540
BaseException]] = asyncio.Queue()
vllm/entrypoints/llm.py
+8
-68
@@ -2,7 +2,6 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
import itertools
5
-import warnings
5
from collections.abc import Sequence
6
from contextlib import contextmanager
7
from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Optional, Union,
@@ -40,15 +39,13 @@ from vllm.inputs import PromptType, SingletonPrompt, TextPrompt, TokensPrompt
39
from vllm.inputs.parse import parse_and_batch_prompt
40
from vllm.logger import init_logger
41
from vllm.lora.request import LoRARequest
43
-from vllm.model_executor.guided_decoding.guided_fields import (
44
- GuidedDecodingRequest, LLMGuidedOptions)
42
from vllm.model_executor.layers.quantization import QuantizationMethods
43
from vllm.outputs import (ClassificationRequestOutput, EmbeddingRequestOutput,
44
PoolingRequestOutput, RequestOutput,
45
ScoringRequestOutput)
46
from vllm.pooling_params import PoolingParams
50
-from vllm.sampling_params import (BeamSearchParams, GuidedDecodingParams,
51
- RequestOutputKind, SamplingParams)
47
+from vllm.sampling_params import (BeamSearchParams, RequestOutputKind,
48
+ SamplingParams)
49
from vllm.tasks import PoolingTask
50
from vllm.transformers_utils.tokenizer import (AnyTokenizer, MistralTokenizer,
51
get_cached_tokenizer)
@@ -330,8 +327,6 @@ class LLM:
327
*,
328
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
329
lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
333
- guided_options_request: Optional[Union[LLMGuidedOptions,
334
- GuidedDecodingRequest]] = None,
330
) -> list[RequestOutput]:
331
...
332
@@ -345,8 +340,6 @@ class LLM:
340
prompt_token_ids: Optional[list[int]] = None,
341
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
342
lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
348
- guided_options_request: Optional[Union[LLMGuidedOptions,
349
- GuidedDecodingRequest]] = None,
343
) -> list[RequestOutput]:
344
...
345
@@ -360,8 +353,6 @@ class LLM:
353
prompt_token_ids: Optional[list[list[int]]] = None,
354
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
355
lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
363
- guided_options_request: Optional[Union[LLMGuidedOptions,
364
- GuidedDecodingRequest]] = None,
356
) -> list[RequestOutput]:
357
...
358
@@ -376,8 +367,6 @@ class LLM:
367
prompt_token_ids: list[int],
368
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
369
lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
379
- guided_options_request: Optional[Union[LLMGuidedOptions,
380
- GuidedDecodingRequest]] = None,
370
) -> list[RequestOutput]:
371
...
372
@@ -392,8 +381,6 @@ class LLM:
381
prompt_token_ids: list[list[int]],
382
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
383
lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
395
- guided_options_request: Optional[Union[LLMGuidedOptions,
396
- GuidedDecodingRequest]] = None,
384
) -> list[RequestOutput]:
385
...
386
@@ -406,8 +393,6 @@ class LLM:
393
prompt_token_ids: Union[list[int], list[list[int]]],
394
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
395
lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
409
- guided_options_request: Optional[Union[LLMGuidedOptions,
410
- GuidedDecodingRequest]] = None,
396
) -> list[RequestOutput]:
397
...
398
@@ -425,8 +410,6 @@ class LLM:
410
prompt_token_ids: Optional[Union[list[int], list[list[int]]]] = None,
411
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
412
lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
428
- guided_options_request: Optional[Union[LLMGuidedOptions,
429
- GuidedDecodingRequest]] = None,
413
priority: Optional[list[int]] = None,
414
) -> list[RequestOutput]:
415
"""Generates the completions for the input prompts.
@@ -478,14 +461,6 @@ class LLM:
461
parsed_prompts = cast(Union[PromptType, Sequence[PromptType]],
462
prompts)
463
481
- if isinstance(guided_options_request, dict):
482
- if len(guided_options_request) > 1:
483
- raise ValueError(
484
- "You can only use one guided decoding but multiple is "
485
- f"specified: {guided_options_request}")
486
- guided_options_request = GuidedDecodingRequest(
487
- **guided_options_request)
488
-
464
if sampling_params is None:
465
# Use default sampling params.
466
sampling_params = self.get_default_sampling_params()
@@ -507,7 +482,6 @@ class LLM:
482
params=sampling_params,
483
use_tqdm=use_tqdm,
484
lora_request=lora_request,
510
- guided_options=guided_options_request,
485
tokenization_kwargs=tokenization_kwargs,
486
priority=priority,
487
)
@@ -1361,17 +1335,17 @@ class LLM:
1335
of your inputs into a single list and pass it to this method.
1336
1337
Supports both text and multi-modal data (images, etc.) when used with
1364
- appropriate multi-modal models. For multi-modal inputs, ensure the
1338
+ appropriate multi-modal models. For multi-modal inputs, ensure the
1339
prompt structure matches the model's expected input format.
1340
1341
Args:
1368
- data_1: Can be a single prompt, a list of prompts or
1369
- `ScoreMultiModalParam`, which can contain either text or
1370
- multi-modal data. When a list, it must have the same length as
1342
+ data_1: Can be a single prompt, a list of prompts or
1343
+ `ScoreMultiModalParam`, which can contain either text or
1344
+ multi-modal data. When a list, it must have the same length as
1345
the `data_2` list.
1372
- data_2: The data to pair with the query to form the input to
1346
+ data_2: The data to pair with the query to form the input to
1347
the LLM. Can be text or multi-modal data. See [PromptType]
1374
- [vllm.inputs.PromptType] for more details about the format of
1348
+ [vllm.inputs.PromptType] for more details about the format of
1349
each prompt.
1350
use_tqdm: If `True`, shows a tqdm progress bar.
1351
If a callable (e.g., `functools.partial(tqdm, leave=False)`),
@@ -1582,17 +1556,8 @@ class LLM:
1556
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
1557
lora_request: Optional[Union[Sequence[LoRARequest], LoRARequest]],
1558
tokenization_kwargs: Optional[dict[str, Any]] = None,
1585
- guided_options: Optional[GuidedDecodingRequest] = None,
1559
priority: Optional[list[int]] = None,
1560
) -> None:
1588
- if guided_options is not None:
1589
- warnings.warn(
1590
- "guided_options_request is deprecated, use "
1591
- "SamplingParams.guided_decoding instead",
1592
- DeprecationWarning,
1593
- stacklevel=2,
1594
- )
1595
-
1561
if isinstance(prompts, (str, dict)):
1562
# Convert a single prompt to a list.
1563
prompts = [prompts]
@@ -1608,8 +1573,6 @@ class LLM:
1573
1574
for sp in params if isinstance(params, Sequence) else (params, ):
1575
if isinstance(sp, SamplingParams):
1611
- self._add_guided_params(sp, guided_options)
1612
-
1576
# We only care about the final output
1577
sp.output_kind = RequestOutputKind.FINAL_ONLY
1578
@@ -1647,29 +1610,6 @@ class LLM:
1610
priority=priority,
1611
)
1612
1650
- def _add_guided_params(
1651
- self,
1652
- params: SamplingParams,
1653
- guided_options: Optional[GuidedDecodingRequest] = None):
1654
- if guided_options is None:
1655
- return params
1656
-
1657
- if params.guided_decoding is not None:
1658
- raise ValueError("Cannot set both guided_options_request and "
1659
- "params.guided_decoding.")
1660
-
1661
- params.guided_decoding = GuidedDecodingParams(
1662
- json=guided_options.guided_json,
1663
- regex=guided_options.guided_regex,
1664
- choice=guided_options.guided_choice,
1665
- grammar=guided_options.guided_grammar,
1666
- json_object=guided_options.guided_json_object,
1667
- backend=guided_options.guided_decoding_backend,
1668
- whitespace_pattern=guided_options.guided_whitespace_pattern,
1669
- structural_tag=guided_options.structural_tag,
1670
- )
1671
- return params
1672
-
1613
def _run_engine(
1614
self,
1615
*,
vllm/model_executor/guided_decoding/__init__.py
deleted
-192
@@ -1,192 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-from __future__ import annotations
5
-
6
-from typing import TYPE_CHECKING
7
-
8
-from vllm.logger import init_logger
9
-from vllm.model_executor.guided_decoding.utils import (
10
- convert_lark_to_gbnf, grammar_is_likely_lark,
11
- has_lmf_unsupported_json_features, has_xgrammar_unsupported_json_features)
12
-from vllm.reasoning import ReasoningParserManager
13
-
14
-if TYPE_CHECKING:
15
- from transformers import PreTrainedTokenizer
16
-
17
- from vllm.config import ModelConfig
18
- from vllm.logits_process import LogitsProcessor
19
- from vllm.sampling_params import GuidedDecodingParams
20
-
21
-logger = init_logger(__name__)
22
-
23
-
24
-def maybe_backend_fallback(
25
- guided_params: GuidedDecodingParams) -> GuidedDecodingParams:
26
-
27
- def fallback_or_error(guided_params: GuidedDecodingParams, message: str,
28
- fallback: str) -> None:
29
- """Change the backend to the specified fallback with a warning log,
30
- or raise a ValueError if the `disable_fallback` option is specified."""
31
- if guided_params.disable_fallback:
32
- raise ValueError(message)
33
-
34
- logger.warning("%s Falling back to use %s instead.", message, fallback)
35
- guided_params.backend = fallback
36
-
37
- # `auto` was added for V1 to explicitly declare a mode that has fallbacks
38
- # in place. If that is specified with V0, treat it as `xgrammar`, as we have
39
- # fallbacks enabled for that and it is the V0 default.
40
- if guided_params.backend == "auto":
41
- guided_params.backend = "xgrammar"
42
-
43
- # lm-format-enforce doesn't support grammar, fallback to xgrammar
44
- if guided_params.backend == "lm-format-enforcer":
45
- if guided_params.grammar is not None:
46
- fallback_or_error(
47
- guided_params,
48
- "lm-format-enforcer does not support grammar guided decoding.",
49
- "xgrammar")
50
-
51
- # lm-format-enforcer doesn't support some JSON schema features
52
- elif (guided_params.json is not None
53
- and has_lmf_unsupported_json_features(guided_params.json)):
54
- fallback_or_error(
55
- guided_params,
56
- "lm-format-enforcer does not support advanced JSON schema "
57
- "features like patterns or numeric ranges.", "outlines")
58
-
59
- if guided_params.backend == "xgrammar":
60
- from vllm.model_executor.guided_decoding.xgrammar_decoding import (
61
- xgr_installed)
62
-
63
- # xgrammar doesn't support some JSON schema features
64
- if (guided_params.json is not None and
65
- has_xgrammar_unsupported_json_features(guided_params.json)):
66
- fallback_or_error(
67
- guided_params,
68
- "xgrammar does not support advanced JSON schema features like "
69
- "string length, item limits, or property bounds.", "outlines")
70
-
71
- # xgrammar only supports GBNF grammars, so we must convert Lark.
72
- # We must check if the grammar is likely Lark and if that
73
- # grammar is convertible to GBNF
74
- elif (guided_params.grammar is not None
75
- and grammar_is_likely_lark(guided_params.grammar)):
76
- try:
77
- convert_lark_to_gbnf(guided_params.grammar)
78
- except Exception:
79
- fallback_or_error(
80
- guided_params,
81
- "xgrammar does not support Lark grammars and the "
82
- "grammar failed to convert to GBNF.", "guidance")
83
-
84
- # If the xgrammar module cannot be imported successfully,
85
- # we should still allow users to use guided decoding with a fallback.
86
- elif not xgr_installed:
87
- fallback_or_error(
88
- guided_params,
89
- "xgrammar module cannot be imported successfully.", "guidance")
90
-
91
- if guided_params.backend == "outlines":
92
- if guided_params.json_object is not None:
93
- # outlines doesn't support json_object, fallback to guidance
94
- fallback_or_error(guided_params,
95
- "outlines does not support json_object.",
96
- "guidance")
97
- elif guided_params.grammar is not None:
98
- # outlines grammar support has been removed, fallback to guidance
99
- # if it is a lark-based grammar and xgrammar otherwise
100
- if grammar_is_likely_lark(guided_params.grammar):
101
- fallback_or_error(guided_params,
102
- "outlines no longer supports grammars.",
103
- "guidance")
104
- else:
105
- # The grammar is likely already GBNF format.
106
- fallback_or_error(guided_params,
107
- "outlines no longer supports grammars.",
108
- "xgrammar")
109
-
110
- return guided_params
111
-
112
-
113
-async def get_guided_decoding_logits_processor(
114
- guided_params: GuidedDecodingParams,
115
- tokenizer: PreTrainedTokenizer,
116
- model_config: ModelConfig,
117
- reasoning_backend: str | None = None) -> LogitsProcessor | None:
118
-
119
- reasoner = None
120
- if reasoning_backend:
121
- reasoner_class = ReasoningParserManager.get_reasoning_parser(
122
- reasoning_backend)
123
- reasoner = reasoner_class(tokenizer)
124
-
125
- guided_params = maybe_backend_fallback(guided_params)
126
-
127
- if guided_params.backend == 'outlines':
128
- # NOTE: lazy import outlines to avoid https://github.com/vllm-project/vllm/issues/4193
129
- from vllm.model_executor.guided_decoding.outlines_decoding import ( # noqa
130
- get_outlines_guided_decoding_logits_processor)
131
- return await get_outlines_guided_decoding_logits_processor(
132
- guided_params, tokenizer, reasoner)
133
- if guided_params.backend == 'lm-format-enforcer':
134
- from vllm.model_executor.guided_decoding.lm_format_enforcer_decoding import ( # noqa
135
- get_local_lm_format_enforcer_guided_decoding_logits_processor)
136
- return get_local_lm_format_enforcer_guided_decoding_logits_processor(
137
- guided_params, tokenizer)
138
- if guided_params.backend == 'xgrammar':
139
- from vllm.model_executor.guided_decoding.xgrammar_decoding import ( # noqa
140
- get_local_xgrammar_guided_decoding_logits_processor)
141
- return get_local_xgrammar_guided_decoding_logits_processor(
142
- guided_params, tokenizer, model_config, reasoner)
143
- if guided_params.backend == 'guidance':
144
- from vllm.model_executor.guided_decoding.guidance_decoding import (
145
- get_local_guidance_guided_decoding_logits_processor)
146
- return get_local_guidance_guided_decoding_logits_processor(
147
- guided_params, tokenizer)
148
- raise ValueError(
149
- f"Unknown guided decoding backend '{guided_params.backend}'. "
150
- "Must be one of 'outlines, 'lm-format-enforcer', 'xgrammar', 'guidance'"
151
- )
152
-
153
-
154
-def get_local_guided_decoding_logits_processor(
155
- guided_params: GuidedDecodingParams,
156
- tokenizer: PreTrainedTokenizer,
157
- model_config: ModelConfig,
158
- reasoning_backend: str | None = None) -> LogitsProcessor | None:
159
- guided_params = maybe_backend_fallback(guided_params)
160
-
161
- reasoner = None
162
- if reasoning_backend:
163
- reasoner_class = ReasoningParserManager.get_reasoning_parser(
164
- reasoning_backend)
165
- reasoner = reasoner_class(tokenizer)
166
-
167
- if guided_params.backend == 'outlines':
168
- # NOTE: lazy import outlines to avoid https://github.com/vllm-project/vllm/issues/4193
169
- from vllm.model_executor.guided_decoding.outlines_decoding import ( # noqa
170
- get_local_outlines_guided_decoding_logits_processor)
171
- return get_local_outlines_guided_decoding_logits_processor(
172
- guided_params, tokenizer, reasoner)
173
- if guided_params.backend == 'lm-format-enforcer':
174
- from vllm.model_executor.guided_decoding.lm_format_enforcer_decoding import ( # noqa
175
- get_local_lm_format_enforcer_guided_decoding_logits_processor)
176
- return get_local_lm_format_enforcer_guided_decoding_logits_processor(
177
- guided_params, tokenizer)
178
- if guided_params.backend == 'xgrammar':
179
- from vllm.model_executor.guided_decoding.xgrammar_decoding import ( # noqa
180
- get_local_xgrammar_guided_decoding_logits_processor)
181
- return get_local_xgrammar_guided_decoding_logits_processor(
182
- guided_params, tokenizer, model_config, reasoner)
183
- if guided_params.backend == 'guidance':
184
- from vllm.model_executor.guided_decoding.guidance_decoding import (
185
- get_local_guidance_guided_decoding_logits_processor)
186
- return get_local_guidance_guided_decoding_logits_processor(
187
- guided_params, tokenizer)
188
-
189
- raise ValueError(
190
- f"Unknown guided decoding backend '{guided_params.backend}'. "
191
- "Must be one of 'outlines, 'lm-format-enforcer', 'xgrammar', 'guidance'"
192
- )
vllm/model_executor/guided_decoding/guidance_decoding.py
deleted
-63
@@ -1,63 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-import json
4
-
5
-import llguidance
6
-from regex import escape as regex_escape
7
-from transformers import PreTrainedTokenizerBase
8
-
9
-from vllm.model_executor.guided_decoding.guidance_logits_processors import (
10
- GuidanceLogitsProcessor)
11
-from vllm.sampling_params import GuidedDecodingParams
12
-from vllm.v1.structured_output.backend_guidance import (
13
- process_for_additional_properties)
14
-
15
-
16
-def get_local_guidance_guided_decoding_logits_processor(
17
- guided_params: GuidedDecodingParams,
18
- tokenizer: PreTrainedTokenizerBase) -> GuidanceLogitsProcessor:
19
- """
20
- Given an OpenAI-compatible request, check for guided decoding parameters
21
- and get the necessary logits processor for the given guide.
22
- """
23
-
24
- grm = ""
25
- any_whitespace = not guided_params.disable_any_whitespace
26
- if (guide_json := guided_params.json) is not None:
27
- # Optionally set additionalProperties to False at the top-level
28
- # By default, other backends do not allow additional top-level
29
- # properties, so this makes guidance more similar to other backends
30
- if guided_params.disable_additional_properties:
31
- if not isinstance(guide_json, str):
32
- guide_json = json.dumps(guide_json)
33
- guide_json = process_for_additional_properties(guide_json)
34
-
35
- grm = llguidance.LLMatcher.grammar_from_json_schema(
36
- guide_json,
37
- overrides={"whitespace_pattern": guided_params.whitespace_pattern},
38
- defaults={
39
- "whitespace_flexible": any_whitespace,
40
- })
41
- elif guided_params.json_object:
42
- grm = llguidance.LLMatcher.grammar_from_json_schema(
43
- '{"type": "object"}',
44
- overrides={"whitespace_pattern": guided_params.whitespace_pattern},
45
- defaults={
46
- "whitespace_flexible": any_whitespace,
47
- })
48
- elif guided_params.regex:
49
- grm = llguidance.grammar_from("regex", guided_params.regex)
50
- elif guided_params.choice:
51
- # choice just uses regex
52
- choices = (regex_escape(str(choice))
53
- for choice in guided_params.choice)
54
- choices_regex = "(" + "|".join(choices) + ")"
55
- grm = llguidance.grammar_from("regex", choices_regex)
56
- elif guided_params.grammar:
57
- # this supports Lark and GBNF
58
- grm = llguidance.grammar_from("grammar", guided_params.grammar)
59
-
60
- if grm:
61
- return GuidanceLogitsProcessor(grm, tokenizer)
62
-
63
- raise ValueError("Unknown guided decoding mode")
vllm/model_executor/guided_decoding/guidance_logits_processors.py
deleted
-104
@@ -1,104 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-import copy
4
-import os
5
-from typing import Any
6
-
7
-import llguidance
8
-import llguidance.hf
9
-import llguidance.torch
10
-import torch
11
-from transformers import PreTrainedTokenizerBase
12
-
13
-from vllm.logger import init_logger
14
-
15
-logger = init_logger(__name__)
16
-
17
-
18
-class GuidanceLogitsProcessor:
19
- """Base Guidance Logits Processor"""
20
-
21
- cached_tokenizers: dict[str, Any] = {}
22
-
23
- def __init__(
24
- self,
25
- grammar: str,
26
- tokenizer: PreTrainedTokenizerBase,
27
- ) -> None:
28
- """Base Guidance Logits Processor
29
-
30
- Args:
31
- grammar (str)
32
- grammar to guide the generation
33
- tokenizer (PreTrainedTokenizerBase)
34
- model's tokenizer
35
- """
36
- self.grammar = grammar
37
- self.tokenizer = tokenizer
38
- self.tokenizer_name = tokenizer.name_or_path
39
- self.ll_tokenizer = None
40
- self.ll_matcher = None
41
- self.bitmask = None
42
- self.new_sampling = False
43
- self.initialized = False
44
-
45
- def clone(self) -> "GuidanceLogitsProcessor":
46
- cloned = copy.copy(self)
47
- if self.initialized:
48
- cloned.ll_matcher = llguidance.LLMatcher(
49
- self.ll_tokenizer, # type: ignore[assignment]
50
- self.grammar,
51
- log_level=int(os.environ.get("LLGUIDANCE_LOG_LEVEL", "1")),
52
- )
53
- self.bitmask = llguidance.torch.allocate_token_bitmask(
54
- 1, self.ll_tokenizer.vocab_size) # type: ignore[attr-defined]
55
- return cloned
56
-
57
- def _initialize(self):
58
- if self.initialized:
59
- return
60
-
61
- ll_tokenizer = self.cached_tokenizers.get(self.tokenizer.name_or_path,
62
- None)
63
- if ll_tokenizer is None:
64
- ll_tokenizer = llguidance.hf.from_tokenizer(self.tokenizer, None)
65
- self.cached_tokenizers[self.tokenizer.name_or_path] = ll_tokenizer
66
-
67
- self.ll_tokenizer = ll_tokenizer
68
- self.ll_matcher = llguidance.LLMatcher(
69
- self.ll_tokenizer,
70
- self.grammar,
71
- log_level=int(os.environ.get("LLGUIDANCE_LOG_LEVEL", "1")),
72
- )
73
-
74
- # create reusable bitmask
75
- self.bitmask = llguidance.torch.allocate_token_bitmask(
76
- 1, self.ll_tokenizer.vocab_size) # type: ignore[attr-defined]
77
-
78
- self.initialized = True
79
-
80
- def __call__(
81
- self,
82
- input_ids: list[int],
83
- scores: torch.Tensor,
84
- ) -> torch.Tensor:
85
- # we initialize the guidance model here
86
- # to avoid pickling ll_tokenizer and ll_interpreter
87
- self._initialize()
88
-
89
- if self.new_sampling and len(input_ids) > 0:
90
- self.ll_matcher.consume_token( # type: ignore[attr-defined]
91
- input_ids[-1])
92
- err = self.ll_matcher.get_error() # type: ignore[attr-defined]
93
- if err:
94
- logger.warning("Error in LLMatcher: %s", err)
95
-
96
- llguidance.torch.fill_next_token_bitmask(self.ll_matcher, self.bitmask,
97
- 0)
98
- llguidance.torch.apply_token_bitmask_inplace(
99
- scores,
100
- self.bitmask.to(scores.device)) # type: ignore[attr-defined]
101
-
102
- self.new_sampling = True
103
-
104
- return scores
vllm/model_executor/guided_decoding/guided_fields.py
deleted
-41
@@ -1,41 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-from dataclasses import dataclass
5
-from typing import Optional, TypedDict, Union
6
-
7
-
8
-# These classes are deprecated, see SamplingParams
9
-class LLMGuidedOptions(TypedDict, total=False):
10
- guided_json: Union[dict, str]
11
- guided_regex: str
12
- guided_choice: list[str]
13
- guided_grammar: str
14
- guided_decoding_backend: str
15
- guided_whitespace_pattern: str
16
- guided_json_object: bool
17
-
18
-
19
-@dataclass
20
-class GuidedDecodingRequest:
21
- """One of the fields will be used to retrieve the logit processor."""
22
- guided_json: Optional[Union[dict, str]] = None
23
- guided_regex: Optional[str] = None
24
- guided_choice: Optional[list[str]] = None
25
- guided_grammar: Optional[str] = None
26
- guided_decoding_backend: Optional[str] = None
27
- guided_whitespace_pattern: Optional[str] = None
28
- guided_json_object: Optional[bool] = None
29
- structural_tag: Optional[str] = None
30
-
31
- def __post_init__(self):
32
- """Validate that some fields are mutually exclusive."""
33
- guide_count = sum(x is not None
34
- for x in (self.guided_json, self.guided_regex,
35
- self.guided_choice, self.guided_grammar,
36
- self.guided_json_object,
37
- self.structural_tag))
38
- if guide_count > 1:
39
- raise ValueError(
40
- "You can only use one kind of guided decoding but multiple are "
41
- f"specified: {self.__dict__}")
vllm/model_executor/guided_decoding/lm_format_enforcer_decoding.py
deleted
-67
@@ -1,67 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-from functools import lru_cache
5
-from json import loads as json_loads
6
-from typing import Optional, Union
7
-
8
-from lmformatenforcer import (CharacterLevelParser, JsonSchemaParser,
9
- RegexParser, StringParser,
10
- TokenEnforcerTokenizerData, UnionParser)
11
-from lmformatenforcer.integrations.vllm import (
12
- build_vllm_logits_processor, build_vllm_token_enforcer_tokenizer_data)
13
-from transformers import PreTrainedTokenizerBase
14
-
15
-from vllm.logits_process import LogitsProcessor
16
-from vllm.sampling_params import GuidedDecodingParams
17
-
18
-
19
-def get_local_lm_format_enforcer_guided_decoding_logits_processor(
20
- guided_params: GuidedDecodingParams,
21
- tokenizer) -> Optional[LogitsProcessor]:
22
- """
23
- Given an OpenAI-compatible request, check for guided decoding parameters
24
- and get the necessary logits processor for the given guide.
25
- We cache logit processors by (guide, tokenizer), and on cache hit
26
- we make a shallow copy to reuse the same underlying FSM.
27
- """
28
-
29
- tokenizer_data = _cached_build_vllm_token_enforcer_tokenizer_data(
30
- tokenizer)
31
- character_level_parser: CharacterLevelParser
32
- if guided_params.json:
33
- schema_dict = _normalize_json_schema_object(guided_params.json)
34
- character_level_parser = JsonSchemaParser(schema_dict)
35
- elif guided_params.choice:
36
- character_level_parser = UnionParser(
37
- [StringParser(choice) for choice in guided_params.choice])
38
- elif guided_params.regex:
39
- character_level_parser = RegexParser(guided_params.regex)
40
- elif guided_params.grammar:
41
- # CFG grammar not supported by LMFE
42
- raise ValueError("Cannot construct a guided decoding logits processor"
43
- " using the grammar option with the"
44
- " lm_format_enforcer backend.")
45
- elif guided_params.json_object:
46
- # None means any json object
47
- character_level_parser = JsonSchemaParser(None)
48
- else:
49
- return None
50
-
51
- logits_processor = build_vllm_logits_processor(tokenizer_data,
52
- character_level_parser)
53
- return logits_processor
54
-
55
-
56
-def _normalize_json_schema_object(schema: Union[str, dict]) -> dict:
57
- if isinstance(schema, str):
58
- return json_loads(schema)
59
- if isinstance(schema, dict):
60
- return schema
61
- raise AssertionError(f"Unsupported schema type {schema}")
62
-
63
-
64
-@lru_cache
65
-def _cached_build_vllm_token_enforcer_tokenizer_data(
66
- tokenizer: PreTrainedTokenizerBase) -> TokenEnforcerTokenizerData:
67
- return build_vllm_token_enforcer_tokenizer_data(tokenizer)
vllm/model_executor/guided_decoding/outlines_decoding.py
deleted
-117
@@ -1,117 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-import asyncio
5
-import concurrent.futures
6
-import os
7
-from enum import Enum
8
-from json import dumps as json_dumps
9
-from typing import Optional, Union
10
-
11
-from regex import escape as regex_escape
12
-from transformers import PreTrainedTokenizerBase
13
-
14
-from vllm.model_executor.guided_decoding.outlines_logits_processors import (
15
- JSONLogitsProcessor, RegexLogitsProcessor)
16
-from vllm.reasoning import ReasoningParser
17
-from vllm.sampling_params import GuidedDecodingParams
18
-
19
-
20
-class GuidedDecodingMode(Enum):
21
- JSON = "json"
22
- REGEX = "regex"
23
- CHOICE = "choice"
24
-
25
-
26
-global_thread_pool = None # used for generating logits processor fsm
27
-
28
-# It's not yet clear that using more provides a benefit, and it could
29
-# potentially starve other processes on the machine. We'll cap this for now and
30
-# adjust later if testing proves it to help overcome a bottleneck.
31
-_MAX_THREADPOOL_WORKERS = 16
32
-
33
-
34
-async def get_outlines_guided_decoding_logits_processor(
35
- guided_params: GuidedDecodingParams, tokenizer: PreTrainedTokenizerBase,
36
- reasoner: Optional[ReasoningParser]
37
-) -> Union[JSONLogitsProcessor, RegexLogitsProcessor, None]:
38
- """
39
- Given an OpenAI-compatible request, check for guided decoding parameters
40
- and get the necessary logits processor for the given guide.
41
- """
42
- global global_thread_pool
43
- guide, mode = _get_guide_and_mode(guided_params)
44
- if not guide or not mode:
45
- return None
46
-
47
- if global_thread_pool is None:
48
- max_workers = os.cpu_count() or 2
49
- if max_workers > _MAX_THREADPOOL_WORKERS:
50
- max_workers = _MAX_THREADPOOL_WORKERS
51
- global_thread_pool = concurrent.futures.ThreadPoolExecutor(
52
- max_workers=max_workers)
53
- loop = asyncio.get_running_loop()
54
- return await loop.run_in_executor(global_thread_pool,
55
- _get_logits_processor, guide, tokenizer,
56
- mode, guided_params.whitespace_pattern,
57
- reasoner)
58
-
59
-
60
-def get_local_outlines_guided_decoding_logits_processor(
61
- guided_params: GuidedDecodingParams, tokenizer: PreTrainedTokenizerBase,
62
- reasoner: Optional[ReasoningParser]
63
-) -> Union[JSONLogitsProcessor, RegexLogitsProcessor, None]:
64
- """
65
- Given an OpenAI-compatible request, check for guided decoding parameters
66
- and get the necessary logits processor for the given guide.
67
- """
68
- guide, mode = _get_guide_and_mode(guided_params)
69
- if not guide or not mode:
70
- return None
71
-
72
- return _get_logits_processor(guide, tokenizer, mode,
73
- guided_params.whitespace_pattern, reasoner)
74
-
75
-
76
-def _get_guide_and_mode(
77
- guided_params: GuidedDecodingParams
78
-) -> Union[tuple[str, GuidedDecodingMode], tuple[None, None]]:
79
- if guided_params.json:
80
- if isinstance(guided_params.json, dict):
81
- # turn dict into hashable string
82
- json = json_dumps(guided_params.json)
83
- else:
84
- json = guided_params.json
85
- return json, GuidedDecodingMode.JSON
86
- elif guided_params.regex:
87
- return guided_params.regex, GuidedDecodingMode.REGEX
88
- elif guided_params.choice:
89
- # choice just uses regex
90
- choices = [
91
- regex_escape(str(choice)) for choice in guided_params.choice
92
- ]
93
- choices_regex = "(" + "|".join(choices) + ")"
94
- return choices_regex, GuidedDecodingMode.CHOICE
95
- elif guided_params.grammar:
96
- raise ValueError(
97
- "The `outlines` guided decoding backend no longer supports grammar "
98
- "guided generation. Please use either the `xgrammar` or `guidance` "
99
- "backend")
100
- else:
101
- return None, None
102
-
103
-
104
-def _get_logits_processor(
105
- guide: str,
106
- tokenizer: PreTrainedTokenizerBase,
107
- mode: GuidedDecodingMode,
108
- whitespace_pattern: Union[str, None],
109
- reasoner: Optional[ReasoningParser],
110
-) -> Union[JSONLogitsProcessor, RegexLogitsProcessor]:
111
- if mode == GuidedDecodingMode.JSON:
112
- return JSONLogitsProcessor(guide, tokenizer, whitespace_pattern,
113
- reasoner)
114
- elif mode == GuidedDecodingMode.REGEX or mode == GuidedDecodingMode.CHOICE:
115
- return RegexLogitsProcessor(guide, tokenizer, reasoner)
116
- else:
117
- raise ValueError(f"Unknown guided decoding mode {mode}")
vllm/model_executor/guided_decoding/outlines_logits_processors.py
deleted
-307
@@ -1,307 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-# SPDX-FileCopyrightText: Copyright 2024-present the Outlines developers
4
-from __future__ import annotations
5
-
6
-import copy
7
-import hashlib
8
-import importlib.metadata
9
-import json
10
-import os
11
-from typing import Optional, Union
12
-
13
-import regex as re
14
-import torch
15
-from cachetools import LRUCache
16
-from diskcache import Cache
17
-from outlines_core import Guide, Index, Vocabulary
18
-from outlines_core.json_schema import build_regex_from_schema
19
-from outlines_core.kernels.torch import (_apply_token_bitmask_inplace_kernel,
20
- allocate_token_bitmask)
21
-from pydantic import BaseModel
22
-from transformers import PreTrainedTokenizerBase
23
-from transformers.file_utils import SPIECE_UNDERLINE
24
-from transformers.models.gpt2.tokenization_gpt2 import bytes_to_unicode
25
-
26
-import vllm.envs as envs
27
-from vllm.logger import init_logger
28
-from vllm.reasoning import ReasoningParser
29
-from vllm.transformers_utils.tokenizer import AnyTokenizer
30
-
31
-logger = init_logger(__name__)
32
-
33
-CACHE = None
34
-
35
-
36
-class BaseLogitsProcessor:
37
-
38
- def __init__(self, guide: Guide, eos_token_id: int,
39
- reasoner: Optional[ReasoningParser]) -> None:
40
- self._guide: Guide = guide
41
- self._eos_token_id: int = eos_token_id
42
- self._reasoner: Optional[ReasoningParser] = reasoner
43
- self._mask: Optional[torch.Tensor] = None
44
-
45
- def __call__(self, input_ids: list[int],
46
- scores: torch.Tensor) -> torch.Tensor:
47
- if self._mask is None:
48
- self._mask = allocate_token_bitmask(scores.size(-1))
49
-
50
- # Skip the structured logits processing if reasoning is not finished.
51
- # reasoner is not None only when `--reasoning-parser` is set.
52
- if self._reasoner is not None and not self._reasoner.is_reasoning_end(
53
- input_ids):
54
- return scores
55
-
56
- # Remove the reasoning tokens from the input_ids
57
- # We need this because our implementation relies on the
58
- # input_ids sequence to store the FSM state.
59
- input_ids = (self._reasoner.extract_content_ids(input_ids)
60
- if self._reasoner is not None else input_ids)
61
-
62
- # Vllm V0 engine has a weird bug where we have to repeat
63
- # the eos token id twice for generation to stop, or at least
64
- # that is what we have to do from here in any case.
65
- # This is a patch until a better solution can be pushed
66
- # to outlines_core
67
- if input_ids and input_ids[-1] != self._eos_token_id:
68
- self._guide.advance(token_id=input_ids[-1], return_tokens=False)
69
-
70
- self._guide.write_mask_into(
71
- data_ptr=self._mask.data_ptr(),
72
- numel=self._mask.numel(),
73
- element_size=self._mask.element_size(),
74
- )
75
-
76
- # Any allowed tokens beyond the length of the scores will
77
- # be ignored by the kernel, taking care of the issue with
78
- # models such as Llama 3.2 Vision with an `<|image|>` token
79
- # with id 128256, but scores.shape == torch.Size([128256])
80
- _apply_token_bitmask_inplace_kernel(
81
- logits=scores.unsqueeze(dim=0),
82
- # mask must be on same device
83
- mask=self._mask.to(scores.device, non_blocking=True))
84
- self._mask.to("cpu", non_blocking=True)
85
-
86
- return scores
87
-
88
- def clone(self) -> BaseLogitsProcessor:
89
- guide = copy.deepcopy(self._guide)
90
- guide.reset()
91
- return BaseLogitsProcessor(guide=guide,
92
- eos_token_id=self._eos_token_id,
93
- reasoner=self._reasoner)
94
-
95
-
96
-class RegexLogitsProcessor(BaseLogitsProcessor):
97
-
98
- @classmethod
99
- def _get_guide(cls, regex_string: str,
100
- tokenizer: PreTrainedTokenizerBase) -> Guide:
101
- global CACHE
102
- if CACHE is None:
103
- CACHE = get_cache()
104
- vocabulary = get_vocabulary(tokenizer) # type: ignore[arg-type]
105
- cache_key = f"{vocabulary._hash}_{regex_string}"
106
- if CACHE is not None and cache_key in CACHE:
107
- return Guide(CACHE[cache_key])
108
-
109
- index = Index(regex_string, vocabulary.inner)
110
-
111
- if CACHE is not None:
112
- CACHE[cache_key] = index
113
-
114
- return Guide(index)
115
-
116
- def __init__(self, regex_string: str, tokenizer: PreTrainedTokenizerBase,
117
- reasoner: Optional[ReasoningParser]) -> None:
118
- super().__init__(
119
- guide=RegexLogitsProcessor._get_guide(regex_string, tokenizer),
120
- eos_token_id=tokenizer.eos_token_id, # type: ignore
121
- reasoner=reasoner)
122
-
123
-
124
-class JSONLogitsProcessor(RegexLogitsProcessor):
125
-
126
- def __init__(self, schema: Union[str, dict, BaseModel],
127
- tokenizer: PreTrainedTokenizerBase,
128
- whitespace_pattern: Union[str, None],
129
- reasoner: Optional[ReasoningParser]) -> None:
130
-
131
- if isinstance(schema, type(BaseModel)):
132
- schema_str = json.dumps(schema.model_json_schema())
133
- elif isinstance(schema, dict):
134
- schema_str = json.dumps(schema)
135
- elif isinstance(schema, str):
136
- schema_str = schema
137
- else:
138
- raise ValueError(
139
- f"Cannot parse schema {schema}. The schema must be either "
140
- f"a Pydantic object, a dictionary or a string that contains "
141
- f"the JSON Schema specification")
142
-
143
- regex_string = build_regex_from_schema(schema_str, whitespace_pattern)
144
- super().__init__(regex_string, tokenizer, reasoner)
145
-
146
-
147
-class OutlinesVocabulary:
148
- """
149
- Wrapper class for `outlines_core.Vocabulary`,
150
- which allows us to store a hash with the vocabulary
151
- """
152
-
153
- def __init__(self, vocabulary: Vocabulary) -> None:
154
- # Actual vocabulary object
155
- self.inner = vocabulary
156
- # Have to do abs(hash()) because python hashes can
157
- # be negative, and we are using hash as a cache key.
158
- hex_str = hashlib.sha256(
159
- vocabulary.__repr__().encode('utf-8')).hexdigest()
160
- hash_int = int(hex_str, 16)
161
- self._hash = hash_int
162
-
163
-
164
-re_llama_byte_token = re.compile(r"^<0x[0-9A-F]{2}>$")
165
-re_replacement_seq = re.compile(r"^.{0,6}�+.{0,6}$")
166
-
167
-
168
-def _reduced_vocabulary(tokenizer: AnyTokenizer,
169
- eos_token_id: int) -> dict[bytes, list[int]]:
170
- """Create a map from vocabulary tokens to lists of equivalent token ids.
171
-
172
- Returns:
173
- A Dict of token string -> equivalent token ids
174
- """
175
- unicode_to_bytes = {v: k for k, v in bytes_to_unicode().items()}
176
-
177
- def convert_token_to_string(token: str) -> str:
178
-
179
- string = tokenizer.convert_tokens_to_string([token])
180
-
181
- # A hack to handle missing spaces to HF's Llama tokenizers
182
- if (type(token) is str and token.startswith(SPIECE_UNDERLINE)
183
- or token == "<0x20>"):
184
- return " " + string
185
-
186
- return string
187
-
188
- vocabulary: dict[bytes, list[int]] = {}
189
- empty_token_ids: list[int] = []
190
- for token, token_idx in tokenizer.get_vocab().items():
191
- if token in tokenizer.all_special_tokens: # type: ignore
192
- continue
193
-
194
- token_str = convert_token_to_string(token)
195
- if token_str:
196
- if isinstance(token, (bytes, bytearray)):
197
- # For BPE tokenizers where tokens are stored as bytes.
198
-
199
- # safe to ignore since token_str is of type (bytearray, bytes)
200
- # by this point.
201
- token_bytes = bytes(token_str) # type: ignore[arg-type]
202
-
203
- elif "\ufffd" in token_str and not re_replacement_seq.match(
204
- token_str):
205
- # Handle tokens with invalid UTF-8 sequences.
206
- if re_llama_byte_token.match(token):
207
- # Llama-like tokenizers use <0xXX> for incomplete sequences.
208
- token_bytes = bytes([int(token[3:5], 16)])
209
- else:
210
- # GPT2 tokenizers: map each byte back using unicode_to_bytes
211
- byte_vals = [unicode_to_bytes.get(c) for c in token]
212
- if None in byte_vals:
213
- raise RuntimeError(
214
- f"Cannot convert token `{token}`"
215
- f" ({token_idx}) to bytes: {token_str}")
216
- # safe to ignore, since if None in byte_vals,
217
- # an error is thrown.
218
- token_bytes = bytes(byte_vals) # type: ignore[arg-type]
219
- else:
220
- token_bytes = token_str.encode('utf-8')
221
-
222
- if token_idx != eos_token_id:
223
- vocabulary.setdefault(token_bytes, []).append(token_idx)
224
- else:
225
- empty_token_ids.append(token_idx)
226
-
227
- return vocabulary
228
-
229
-
230
-def get_vocabulary(tokenizer: AnyTokenizer) -> Vocabulary:
231
- """Get the `Vocabulary` object for a given tokenizer.
232
- """
233
- if hasattr(tokenizer, "_outlines_vocabulary"):
234
- return tokenizer._outlines_vocabulary # type: ignore
235
-
236
- try:
237
- if hasattr(
238
- tokenizer,
239
- "eos_token_id",
240
- ) and tokenizer.eos_token_id is not None:
241
- eos_token_id = tokenizer.eos_token_id
242
- else:
243
- raise ValueError(
244
- f"Error during guided decoding setup: Tokenizer"
245
- f" ({type(tokenizer)}) has no `eos_token_id` property, "
246
- "but `eos_token_id` is required for guided decoding"
247
- " to work properly.")
248
-
249
- reduced_vocab = _reduced_vocabulary(
250
- tokenizer,
251
- eos_token_id #type: ignore
252
- )
253
- vocabulary = OutlinesVocabulary(Vocabulary(eos_token_id,
254
- reduced_vocab))
255
- tokenizer._outlines_vocabulary = vocabulary # type: ignore
256
-
257
- return vocabulary
258
- except AttributeError as e:
259
- raise ValueError(f"Cannot get the vocabulary of the tokenizer "
260
- f"({type(tokenizer)}). The tokenizer should have a "
261
- "get_vocab method.") from e
262
-
263
-
264
-def get_cache_path() -> str:
265
- """Get the context object that contains previously-computed return values"""
266
- outlines_cache_dir = os.getenv("OUTLINES_CACHE_DIR")
267
- xdg_cache_home = os.getenv("XDG_CACHE_HOME")
268
- home_dir = os.path.expanduser("~")
269
-
270
- if outlines_cache_dir:
271
- # OUTLINES_CACHE_DIR takes precedence
272
- return outlines_cache_dir
273
- elif xdg_cache_home:
274
- return os.path.join(xdg_cache_home, ".cache", "outlines")
275
- # If homedir is "/", we may be inside a container, and thus writing to
276
- # root would be problematic, so we fallback to using a tempfile.
277
- # Also validate the path exists, since os.path.expanduser does
278
- # not garuntee existence.
279
- elif os.path.isdir(home_dir) and home_dir != "/":
280
- # Default Unix fallback: ~/.cache/outlines
281
- return os.path.join(home_dir, ".cache", "outlines")
282
- else:
283
- import tempfile
284
-
285
- # home_dir may be / inside a docker container without existing user
286
- tempdir = tempfile.gettempdir()
287
- return os.path.join(tempdir, ".cache", "outlines")
288
-
289
-
290
-def get_cache():
291
- """Get the Cache instance to be used for index caching"""
292
-
293
- cache_dir = get_cache_path()
294
- if envs.VLLM_V0_USE_OUTLINES_CACHE:
295
- logger.warning("Enabling outlines cache. This is an unbounded on-disk "
296
- "cache. It may consume a lot of disk space and should "
297
- "not be used with untrusted clients.")
298
- cache = Cache(cache_dir, eviction_policy="none", cull_limit=0)
299
- outlines_version = importlib.metadata.version("outlines_core")
300
-
301
- cached_version = cache.get('__version__', None)
302
- if cached_version != outlines_version:
303
- cache.clear()
304
- cache.set('__version__', outlines_version)
305
- return cache
306
- else:
307
- return LRUCache(maxsize=128)
vllm/model_executor/guided_decoding/utils.py
deleted
-242
@@ -1,242 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-import regex as re
5
-
6
-
7
-def has_xgrammar_unsupported_json_features(schema: dict) -> bool:
8
- """Check if JSON schema contains features unsupported by xgrammar."""
9
-
10
- def check_object(obj: dict) -> bool:
11
- if not isinstance(obj, dict):
12
- return False
13
-
14
- # Check for numeric ranges
15
- if obj.get("type") in ("integer", "number") and ("multipleOf" in obj):
16
- return True
17
-
18
- # Check for array unsupported keywords
19
- if obj.get("type") == "array" and any(key in obj for key in [
20
- "uniqueItems", "contains", "minContains", "maxContains",
21
- "minItems", "maxItems"
22
- ]):
23
- return True
24
-
25
- # Unsupported keywords for strings
26
- if obj.get("type") == "string" and any(
27
- key in obj for key in ["minLength", "maxLength", "format"]):
28
- return True
29
-
30
- # Unsupported keywords for objects
31
- if obj.get("type") == "object" and any(key in obj for key in [
32
- "minProperties", "maxProperties", "propertyNames",
33
- "patternProperties"
34
- ]):
35
- return True
36
-
37
- # Recursively check all nested objects and arrays
38
- for value in obj.values():
39
- if isinstance(value, dict):
40
- if check_object(value):
41
- return True
42
- elif isinstance(value, list):
43
- for item in value:
44
- if isinstance(item, dict) and check_object(item):
45
- return True
46
-
47
- return False
48
-
49
- return check_object(schema)
50
-
51
-
52
-def has_lmf_unsupported_json_features(schema: dict) -> bool:
53
- """
54
- Check if JSON schema contains features unsupported
55
- by lm_format_enforcer.
56
-
57
- Known issues:
58
- - Regex patterns:
59
- "grade": {
60
- "type": "string",
61
- "pattern": "^[A-D]$" # Regex pattern
62
- },
63
- """
64
-
65
- def check_object(obj: dict) -> bool:
66
- if not isinstance(obj, dict):
67
- return False
68
-
69
- # Check for pattern restrictions
70
- if "pattern" in obj:
71
- return True
72
-
73
- # Recursively check all nested objects and arrays
74
- for value in obj.values():
75
- if isinstance(value, dict):
76
- if check_object(value):
77
- return True
78
- elif isinstance(value, list):
79
- for item in value:
80
- if isinstance(item, dict) and check_object(item):
81
- return True
82
-
83
- return False
84
-
85
- return check_object(schema)
86
-
87
-
88
-def grammar_is_likely_lark(grammar_str: str) -> bool:
89
- """
90
- Check if grammar appears to use Lark syntax.
91
-
92
- Args:
93
- grammar_str: Input grammar string
94
-
95
- Returns:
96
- bool: True if grammar appears to be in Lark format, False otherwise
97
-
98
- Examples:
99
- >>> grammar_is_likely_lark("rule: 'abc'")
100
- True
101
- >>> grammar_is_likely_lark("rule ::= 'abc'")
102
- False
103
- """
104
- if not grammar_str or not isinstance(grammar_str, str):
105
- return False
106
-
107
- for line in grammar_str.split('\n'):
108
- # Remove both comment styles
109
- line = re.sub(r'(#|//).*$', '', line).strip()
110
- if not line:
111
- continue
112
-
113
- # Look for GBNF rule definition
114
- if '::=' in line:
115
- return False
116
-
117
- return True
118
-
119
-
120
-def convert_lark_to_gbnf(grammar_str: str) -> str:
121
- """
122
- Convert a Lark grammar string to GBNF format.
123
-
124
- GBNF reference:
125
- https://github.com/ggerganov/llama.cpp/blob/master/grammars/README.md
126
- Lark grammar reference:
127
- https://lark-parser.readthedocs.io/en/latest/grammar.html
128
-
129
- Args:
130
- grammar_str: Input grammar in Lark format
131
-
132
- Returns:
133
- str: Converted grammar in GBNF format
134
-
135
- Examples:
136
- >>> print(convert_lark_to_gbnf("rule: 'hello'"))
137
- root ::= rule
138
- rule ::= "hello"
139
- """
140
- if not isinstance(grammar_str, str):
141
- raise ValueError(f"Grammar must be a string, got {type(grammar_str)}")
142
- if not grammar_str.strip():
143
- raise ValueError("Grammar string cannot be empty")
144
-
145
- defined_rules = set()
146
- referenced_rules = set()
147
- output_lines = []
148
-
149
- def clean_line(line: str) -> str:
150
- """Remove comments and whitespace from line."""
151
- return re.sub(r'(#|//).*$', '', line).strip()
152
-
153
- def check_quotes(text: str, rule_name: str, line_num: int) -> None:
154
- """Validate quote matching in text."""
155
- if text.count("'") % 2 != 0 or text.count('"') % 2 != 0:
156
- raise ValueError(
157
- f"Mismatched quotes in {rule_name} on line {line_num}")
158
-
159
- def extract_references(text: str) -> set:
160
- """Extract rule references from text."""
161
- # Remove quoted strings and special characters
162
- text = re.sub(r'"[^"]*"', '', text)
163
- text = re.sub(r'[+*?()|\[\]{}]', ' ', text)
164
- return set(re.findall(r'\b[a-zA-Z_][a-zA-Z0-9_]*\b', text))
165
-
166
- # First pass: Find root rule and validate rule definitions
167
- lines = [clean_line(line) for line in grammar_str.split('\n')]
168
- first_rule = None
169
-
170
- for line_num, line in enumerate(lines, 1):
171
- if not line or line.startswith('|'):
172
- continue
173
-
174
- if ':' in line:
175
- try:
176
- name = line.split(':', 1)[0].strip().strip('?')
177
- defined_rules.add(name)
178
- if first_rule is None:
179
- first_rule = name
180
- if name == 'start':
181
- first_rule = 'start'
182
- except IndexError as e:
183
- raise ValueError(f"Invalid rule format on line {line_num}. "
184
- "Expected 'rule_name: definition'") from e
185
-
186
- if not defined_rules:
187
- raise ValueError("No valid rules found in grammar")
188
-
189
- # Add root rule
190
- output_lines.append(f"root ::= {first_rule}")
191
-
192
- # Second pass: Process rule definitions and alternatives
193
- current_rule = None
194
- current_definition = []
195
-
196
- for line_num, line in enumerate(lines, 1):
197
- if not line:
198
- continue
199
-
200
- try:
201
- if ':' in line and not line.startswith('|'):
202
- # Save previous rule if exists
203
- if current_rule:
204
- output_lines.append(
205
- f"{current_rule} ::= {' | '.join(current_definition)}")
206
-
207
- # Process new rule
208
- name, definition = line.split(':', 1)
209
- current_rule = name.strip().strip('?')
210
-
211
- check_quotes(definition, f"rule '{current_rule}'", line_num)
212
- definition = re.sub(r"'([^']*)'", r'"\1"', definition)
213
- referenced_rules.update(extract_references(definition))
214
- current_definition = [definition.strip()]
215
-
216
- elif line.startswith('|'):
217
- if not current_rule:
218
- raise ValueError(f"Alternative '|' on line {line_num} "
219
- "without a preceding rule definition")
220
-
221
- alt_def = line[1:].strip()
222
- check_quotes(alt_def, f"alternative for rule '{current_rule}'",
223
- line_num)
224
- alt_def = re.sub(r"'([^']*)'", r'"\1"', alt_def)
225
- referenced_rules.update(extract_references(alt_def))
226
- current_definition.append(alt_def)
227
-
228
- except ValueError as e:
229
- raise ValueError(f"Error on line {line_num}: {str(e)}") from e
230
-
231
- # Add final rule if exists
232
- if current_rule:
233
- output_lines.append(
234
- f"{current_rule} ::= {' | '.join(current_definition)}")
235
-
236
- # Validate all rules are defined
237
- undefined_rules = referenced_rules - defined_rules - {'root'}
238
- if undefined_rules:
239
- raise ValueError("Referenced rules are not defined: "
240
- f"{', '.join(sorted(undefined_rules))}")
241
-
242
- return '\n'.join(output_lines)
vllm/model_executor/guided_decoding/xgrammar_decoding.py
deleted
-426
@@ -1,426 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-# noqa: UP007
5
-from __future__ import annotations
6
-
7
-import json
8
-from dataclasses import dataclass, field
9
-from typing import TYPE_CHECKING, Any
10
-
11
-import regex as re
12
-import torch
13
-
14
-import vllm.envs
15
-from vllm.logger import init_logger
16
-
17
-try:
18
- import xgrammar as xgr
19
- xgr_installed = True
20
-except ImportError:
21
- xgr_installed = False
22
- pass
23
-
24
-from vllm.model_executor.guided_decoding.utils import (convert_lark_to_gbnf,
25
- grammar_is_likely_lark)
26
-from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer
27
-
28
-if TYPE_CHECKING:
29
- from transformers import PreTrainedTokenizer
30
-
31
- from vllm.config import ModelConfig
32
- from vllm.reasoning import ReasoningParser
33
- from vllm.sampling_params import GuidedDecodingParams
34
-
35
-logger = init_logger(__name__)
36
-
37
-
38
-def get_local_xgrammar_guided_decoding_logits_processor(
39
- guided_params: GuidedDecodingParams,
40
- tokenizer: PreTrainedTokenizer,
41
- model_config: ModelConfig,
42
- reasoner: ReasoningParser | None,
43
- max_threads: int = 8):
44
- config = GrammarConfig.from_guided_params(guided_params=guided_params,
45
- model_config=model_config,
46
- tokenizer=tokenizer,
47
- max_threads=max_threads)
48
- return XGrammarLogitsProcessor(config, reasoner)
49
-
50
-
51
-@dataclass(frozen=True)
52
-class TokenizerData:
53
- """Immutable container for cached tokenizer data."""
54
- metadata: str
55
- encoded_vocab: list[str] = field(default_factory=list)
56
-
57
-
58
-class TokenizerDataCache:
59
- """Cache manager for tokenizer data to avoid repeated processing."""
60
- _cache: dict[int, TokenizerData] = {}
61
-
62
- @classmethod
63
- def get_tokenizer_data(
64
- cls,
65
- tokenizer: PreTrainedTokenizer,
66
- /,
67
- *,
68
- tokenizer_hash: int,
69
- vocab_size: int,
70
- ) -> TokenizerData:
71
-
72
- if tokenizer_hash not in cls._cache:
73
- tokenizer_info = xgr.TokenizerInfo.from_huggingface(
74
- tokenizer,
75
- # NOTE: We will need to use lm_head's vocab_size
76
- # to determine correct special_token_ids for this tokenizer.
77
- # See https://github.com/mlc-ai/xgrammar/commit/70c959fb6d9cea75aae33c414763cd0602022d92 # noqa: E501
78
- vocab_size=vocab_size,
79
- )
80
- metadata = json.loads(tokenizer_info.dump_metadata())
81
-
82
- # Vendored from xgrammar logic to get encoded_vocab
83
- # https://github.com/mlc-ai/xgrammar/blob/989222175c2a30fb7987d8bcce35bec1bf6817f2/python/xgrammar/tokenizer_info.py#L127 # noqa: E501
84
- try:
85
- vocab_dict = tokenizer.get_vocab()
86
- except AttributeError as e:
87
- raise ValueError(
88
- f"Cannot get the vocabulary of the tokenizer "
89
- f"{type(tokenizer)}. The tokenizer should have a "
90
- "get_vocab method.") from e
91
-
92
- # maintain tokenizer's indexing
93
- encoded_vocab = [""] * tokenizer_info.vocab_size
94
- for token, idx in vocab_dict.items():
95
- if idx < tokenizer_info.vocab_size:
96
- encoded_vocab[idx] = token
97
-
98
- if isinstance(tokenizer, MistralTokenizer):
99
- # REF: https://github.com/mlc-ai/xgrammar/blob/5e141f6ff1ca02bc31f9e512e68b61f2a8ae88e5/tests/python/test_tokenizer_info.py#L43 # noqa: E501
100
- metadata.update({
101
- "vocab_type": xgr.VocabType.BYTE_FALLBACK,
102
- "add_prefix_space": True
103
- })
104
-
105
- cls._cache[tokenizer_hash] = TokenizerData(
106
- encoded_vocab=encoded_vocab,
107
- metadata=json.dumps(metadata),
108
- )
109
-
110
- return cls._cache[tokenizer_hash]
111
-
112
-
113
-class GrammarCompilerCache:
114
- """
115
- Cache for GrammarCompiler instances based on tokenizer.
116
-
117
- This cache reduces the overhead of creating new compiler instances when
118
- using the same tokenizer configuration.
119
- """
120
- _cache: dict[str, xgr.GrammarCompiler] = {}
121
-
122
- @classmethod
123
- def get_compiler(cls, config: GrammarConfig) -> xgr.GrammarCompiler:
124
- cache_key = str(config.tokenizer_hash)
125
-
126
- if cache_key not in cls._cache:
127
- config_data = config.tokenizer_data
128
-
129
- # In TokenizerDataCache.get_tokenizer_data, a serializable
130
- # tokenizer_data is created and cached. This data is used to build
131
- # a tokenizer_info and create an xgrammar compiler.
132
- tokenizer_info = xgr.TokenizerInfo.from_vocab_and_metadata(
133
- encoded_vocab=config_data.encoded_vocab,
134
- metadata=config_data.metadata,
135
- )
136
- cache_size = vllm.envs.VLLM_XGRAMMAR_CACHE_MB * 1024 * 1024
137
- cls._cache[cache_key] = xgr.GrammarCompiler(
138
- tokenizer_info,
139
- max_threads=config.max_threads,
140
- cache_enabled=True,
141
- cache_limit_bytes=cache_size,
142
- )
143
-
144
- return cls._cache[cache_key]
145
-
146
-
147
-@dataclass
148
-class GrammarConfig:
149
- """Serializable configuration for grammar compilation"""
150
- tokenizer_hash: int
151
- tokenizer_data: TokenizerData
152
- json_str: str | None = None
153
- grammar_str: str | None = None
154
- json_object: bool | None = None
155
- any_whitespace: bool = True
156
- regex_str: str | None = None
157
- max_threads: int = 8
158
-
159
- @classmethod
160
- def from_guided_params(cls,
161
- guided_params: GuidedDecodingParams,
162
- model_config: ModelConfig,
163
- tokenizer: PreTrainedTokenizer,
164
- max_threads: int = 8) -> GrammarConfig:
165
-
166
- tokenizer_hash = hash(tokenizer)
167
- tokenizer_data = TokenizerDataCache.get_tokenizer_data(
168
- tokenizer,
169
- tokenizer_hash=tokenizer_hash,
170
- vocab_size=model_config.hf_text_config.vocab_size,
171
- )
172
-
173
- if guided_params.json:
174
- if not isinstance(guided_params.json, str):
175
- json_str = json.dumps(guided_params.json)
176
- else:
177
- json_str = guided_params.json
178
-
179
- any_whitespace = not guided_params.disable_any_whitespace
180
-
181
- # Check and log if model with xgrammar and whitespace have history
182
- # of runaway generation of whitespaces.
183
- # References:
184
- # https://github.com/vllm-project/vllm/pull/12744
185
- # https://github.com/mlc-ai/xgrammar/issues/212
186
- model_with_warn = None
187
-
188
- if 'Mistral' in model_config.model:
189
- model_with_warn = 'Mistral'
190
- elif 'Qwen' in model_config.model:
191
- model_with_warn = 'Qwen'
192
-
193
- if model_with_warn is not None and any_whitespace:
194
- logger.info_once(
195
- "%s model detected, consider setting `disable_any_whitespace` to prevent runaway generation of whitespaces.", # noqa: E501
196
- model_with_warn,
197
- )
198
- # Validate the schema and raise ValueError here if it is invalid.
199
- # This is to avoid exceptions in model execution, which will crash
200
- # the engine worker process.
201
- try:
202
- xgr.Grammar.from_json_schema(json_str,
203
- any_whitespace=any_whitespace)
204
- except RuntimeError as err:
205
- raise ValueError(str(err)) from err
206
-
207
- return cls(json_str=json_str,
208
- tokenizer_hash=tokenizer_hash,
209
- max_threads=max_threads,
210
- tokenizer_data=tokenizer_data,
211
- any_whitespace=any_whitespace)
212
- elif guided_params.grammar:
213
- # XGrammar only supports GBNF grammars, so we must convert Lark
214
- if grammar_is_likely_lark(guided_params.grammar):
215
- try:
216
- grammar_str = convert_lark_to_gbnf(guided_params.grammar)
217
- except ValueError as e:
218
- raise ValueError(
219
- "Failed to convert the grammar from Lark to GBNF. "
220
- "Please either use GBNF grammar directly or specify"
221
- " --guided-decoding-backend=outlines.\n"
222
- f"Conversion error: {str(e)}") from e
223
- else:
224
- grammar_str = guided_params.grammar
225
-
226
- # Validate the grammar and raise ValueError here if it is invalid.
227
- # This is to avoid exceptions in model execution, which will crash
228
- # the engine worker process.
229
- try:
230
- xgr.Grammar.from_ebnf(grammar_str)
231
- except RuntimeError as err:
232
- raise ValueError(str(err)) from err
233
-
234
- return cls(grammar_str=grammar_str,
235
- tokenizer_hash=tokenizer_hash,
236
- max_threads=max_threads,
237
- tokenizer_data=tokenizer_data)
238
- elif guided_params.json_object:
239
- return cls(
240
- json_object=True,
241
- tokenizer_hash=tokenizer_hash,
242
- max_threads=max_threads,
243
- tokenizer_data=tokenizer_data,
244
- )
245
- elif guided_params.choice:
246
- choice_str = GrammarConfig.choice_as_grammar(guided_params.choice)
247
- try:
248
- xgr.Grammar.from_ebnf(choice_str)
249
- except RuntimeError as err:
250
- raise ValueError(str(err)) from err
251
-
252
- return cls(
253
- grammar_str=choice_str,
254
- tokenizer_hash=tokenizer_hash,
255
- max_threads=max_threads,
256
- tokenizer_data=tokenizer_data,
257
- )
258
- elif guided_params.regex:
259
- return cls(
260
- regex_str=guided_params.regex,
261
- tokenizer_hash=tokenizer_hash,
262
- max_threads=max_threads,
263
- tokenizer_data=tokenizer_data,
264
- )
265
- else:
266
- raise ValueError(
267
- "Currently only support JSON and EBNF grammar mode for xgrammar"
268
- )
269
-
270
- @staticmethod
271
- def escape_ebnf_string(s: str) -> str:
272
- """Escape special characters in a EBNF string."""
273
- # Escape double quotes and backslashes
274
- return re.sub(r'(["\\])', r'\\\1', s)
275
-
276
- @staticmethod
277
- def choice_as_grammar(choice: list[str] | None) -> str:
278
- if choice is None:
279
- raise ValueError("Choice is not set")
280
- escaped_choices = (GrammarConfig.escape_ebnf_string(c) for c in choice)
281
- grammar = ('root ::= ' + ' | '.join(f'"{c}"' for c in escaped_choices))
282
- return grammar
283
-
284
- @staticmethod
285
- def tokenizer_info(tokenizer_data: TokenizerData) -> xgr.TokenizerInfo:
286
- return xgr.TokenizerInfo.from_vocab_and_metadata(
287
- encoded_vocab=tokenizer_data.encoded_vocab,
288
- metadata=tokenizer_data.metadata,
289
- )
290
-
291
-
292
-@dataclass
293
-class XGrammarLogitsProcessor:
294
- """Wrapper class to support pickle protocol"""
295
- config: GrammarConfig
296
- reasoner: ReasoningParser | None = None
297
-
298
- ctx: xgr.CompiledGrammar | None = None
299
- tokenizer_info: xgr.TokenizerInfo = None # type: ignore[assignment]
300
- token_bitmask: torch.Tensor = None # type: ignore[assignment]
301
- matchers: list[xgr.GrammarMatcher] = field(default_factory=list)
302
- batch_size: int = field(default=1)
303
- prefilled: bool = field(default=False)
304
-
305
- def __post_init__(self):
306
- if self.tokenizer_info is None:
307
- self.tokenizer_info = self.config.tokenizer_info(
308
- self.config.tokenizer_data)
309
-
310
- def __getstate__(self) -> dict[str, Any]:
311
- return {'config': self.config, 'reasoner': self.reasoner}
312
-
313
- def __setstate__(self, state: dict[str, Any]):
314
- self.config = state['config']
315
- self.reasoner = state['reasoner']
316
-
317
- self.tokenizer_info = GrammarConfig.tokenizer_info(
318
- self.config.tokenizer_data)
319
- self.ctx = None
320
- self.matchers = []
321
- self.batch_size = 1
322
- self.token_bitmask = None # type: ignore[assignment]
323
- self.prefilled = False
324
-
325
- def _ensure_ctx(self):
326
- """Lazily initialize the processor in the worker process"""
327
- if self.ctx is None:
328
- compiler = GrammarCompilerCache.get_compiler(self.config)
329
- if self.config.json_str is not None:
330
- any_whitespace = self.config.any_whitespace
331
- self.ctx = compiler\
332
- .compile_json_schema(self.config.json_str,
333
- any_whitespace=any_whitespace)
334
- elif self.config.grammar_str is not None:
335
- self.ctx = compiler.compile_grammar(self.config.grammar_str)
336
- elif self.config.json_object:
337
- any_whitespace = self.config.any_whitespace
338
- self.ctx = compiler\
339
- .compile_json_schema('{"type": "object"}',
340
- any_whitespace=any_whitespace)
341
- elif self.config.regex_str:
342
- self.ctx = compiler.compile_regex(self.config.regex_str)
343
- else:
344
- raise ValueError(
345
- "Invalid configuration for xgrammar logits processor")
346
-
347
- def __call__(self, input_ids: list[int],
348
- scores: torch.Tensor) -> torch.Tensor:
349
-
350
- # Skip the structured logits processing if reasoning is not finished.
351
- # reasoner is not None only when `--reasoning-parser` is set.
352
- if self.reasoner is not None and \
353
- not self.reasoner.is_reasoning_end(
354
- input_ids):
355
- return scores
356
-
357
- if self.ctx is None:
358
- self._ensure_ctx()
359
-
360
- if len(self.matchers) == 0:
361
- self.matchers = [
362
- xgr.GrammarMatcher(self.ctx) for _ in range(self.batch_size)
363
- ]
364
- self.token_bitmask = xgr.allocate_token_bitmask(
365
- self.batch_size, self.tokenizer_info.vocab_size)
366
-
367
- if not self.prefilled:
368
- # Have not sampled a token yet
369
- self.prefilled = True
370
- else:
371
- for i, matcher in enumerate(self.matchers):
372
- if not matcher.is_terminated():
373
- sampled_token = input_ids[-1]
374
- assert self.matchers[i].accept_token(sampled_token)
375
-
376
- for i, matcher in enumerate(self.matchers):
377
- if not matcher.is_terminated():
378
- # @ubospica: ideally, fill_next_token_bitmask should be
379
- # parallelized with model decoding
380
- # See https://github.com/vllm-project/vllm/pull/10785/files#r1864278303
381
- matcher.fill_next_token_bitmask(self.token_bitmask, i)
382
-
383
- # token_bitmask is a CPU tensor for use with accept_token and
384
- # fill_next_token_bitmask so we move it to the device of scores
385
- device_type = scores.device.type
386
- dtype = scores.dtype
387
- if device_type != "cuda":
388
- # xgrammar on cpu only supports float32 scores
389
- # see: https://github.com/mlc-ai/xgrammar/blob/c1b64920cad24f44f235778c1c00bb52d57da01a/python/xgrammar/kernels/apply_token_bitmask_inplace_cpu.py#L22
390
- scores = scores.to("cpu").float().unsqueeze(0)
391
-
392
- # Note: In this method, if the tensors have different dimensions
393
- # on CPU device fails, but on GPU it runs without error. Hence the
394
- # unsqueeze above for scores, to match the token bitmask shape
395
- xgr.apply_token_bitmask_inplace(
396
- scores, self.token_bitmask.to(scores.device, non_blocking=True))
397
- if device_type != "cuda":
398
- scores = scores.to(dtype).to(device_type).squeeze()
399
-
400
- return scores
401
-
402
- def clone(self) -> XGrammarLogitsProcessor:
403
- """Create a new instance with shared compiled grammar
404
- but separate state"""
405
- new_processor = XGrammarLogitsProcessor(self.config, self.reasoner,
406
- None, self.tokenizer_info)
407
-
408
- # Share the compiled grammar context (immutable after compilation)
409
- new_processor.ctx = self.ctx
410
-
411
- # Create fresh matchers for the new sequence
412
- if self.ctx is not None:
413
- new_processor.matchers = [
414
- xgr.GrammarMatcher(self.ctx) for _ in range(self.batch_size)
415
- ]
416
-
417
- # Create a new token bitmask with the same size
418
- if hasattr(self, 'token_bitmask') and self.token_bitmask is not None:
419
- new_processor.token_bitmask = self.token_bitmask
420
-
421
- # Copy simple attributes
422
- new_processor.batch_size = self.batch_size
423
- # Reset prefilled state for new sequence
424
- new_processor.prefilled = False
425
-
426
- return new_processor