A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
LM Format Enforcer Guided Decoding Support (#3868)
Co-authored-by: Simon Mo <simon.mo@hey.com>
Noam Gat committed
Apr 16, 2024 at 08:54 UTC
05434764cd99990035779cf9a4ed86623b528825
13 files changed
+304
-87
requirements-common.txt
+1
@@ -11,6 +11,7 @@ uvicorn[standard]
11
pydantic >= 2.0 # Required for OpenAI server.
12
prometheus_client >= 0.18.0
13
tiktoken == 0.6.0 # Required for DBRX tokenizer
14
+lm-format-enforcer == 0.9.3
15
outlines == 0.0.34 # Requires torch >= 2.1.0
16
typing_extensions
17
filelock >= 3.10.4 # filelock starts to support `mode` argument from 3.10.4
tests/entrypoints/test_guided_processors.py
+39
-3
@@ -1,11 +1,14 @@
1
# This unit test should be moved to a new
2
# tests/test_guided_decoding directory.
3
-
3
+import pytest
4
import torch
5
from transformers import AutoTokenizer
6
7
-from vllm.model_executor.guided_logits_processors import (JSONLogitsProcessor,
8
- RegexLogitsProcessor)
7
+from vllm.entrypoints.openai.protocol import CompletionRequest
8
+from vllm.model_executor.guided_decoding import (
9
+ get_guided_decoding_logits_processor)
10
+from vllm.model_executor.guided_decoding.outlines_logits_processors import (
11
+ JSONLogitsProcessor, RegexLogitsProcessor)
12
13
TEST_SCHEMA = {
14
"type": "object",
@@ -73,3 +76,36 @@ def test_guided_logits_processors():
76
json_LP(token_ids, tensor)
77
assert tensor.shape == original_tensor.shape
78
assert not torch.allclose(tensor, original_tensor)
79
+
80
+
81
+@pytest.mark.asyncio
82
+@pytest.mark.parametrize("backend", ["outlines", "lm-format-enforcer"])
83
+async def test_guided_logits_processor_black_box(backend: str):
84
+ tokenizer = AutoTokenizer.from_pretrained('HuggingFaceH4/zephyr-7b-beta')
85
+ token_ids = tokenizer.encode(
86
+ f"Give an example IPv4 address with this regex: {TEST_REGEX}")
87
+ regex_request = CompletionRequest(model='test',
88
+ prompt=token_ids,
89
+ guided_regex=TEST_REGEX)
90
+ regex_lp = await get_guided_decoding_logits_processor(
91
+ backend, regex_request, tokenizer)
92
+ assert regex_lp is not None
93
+ tensor = torch.rand(32000)
94
+ original_tensor = torch.clone(tensor)
95
+ tensor = regex_lp(token_ids, tensor)
96
+ assert tensor.shape == original_tensor.shape
97
+ assert not torch.allclose(tensor, original_tensor)
98
+
99
+ token_ids = tokenizer.encode(
100
+ f"Give an employee profile that fits this schema: {TEST_SCHEMA}")
101
+ json_request = CompletionRequest(model='test',
102
+ prompt=token_ids,
103
+ guided_json=TEST_SCHEMA)
104
+ json_lp = await get_guided_decoding_logits_processor(
105
+ backend, json_request, tokenizer)
106
+ assert json_lp is not None
107
+ tensor = torch.rand(32000)
108
+ original_tensor = torch.clone(tensor)
109
+ tensor = json_lp(token_ids, tensor)
110
+ assert tensor.shape == original_tensor.shape
111
+ assert not torch.allclose(tensor, original_tensor)
tests/entrypoints/test_openai_server.py
+50
-19
@@ -506,7 +506,10 @@ async def test_logits_bias(server, client: openai.AsyncOpenAI):
506
assert first_response != completion.choices[0].text
507
508
509
-async def test_guided_json_completion(server, client: openai.AsyncOpenAI):
509
+@pytest.mark.parametrize("guided_decoding_backend",
510
+ ["outlines", "lm-format-enforcer"])
511
+async def test_guided_json_completion(server, client: openai.AsyncOpenAI,
512
+ guided_decoding_backend: str):
513
completion = await client.completions.create(
514
model=MODEL_NAME,
515
prompt=f"Give an example JSON for an employee profile "
@@ -514,7 +517,8 @@ async def test_guided_json_completion(server, client: openai.AsyncOpenAI):
517
n=3,
518
temperature=1.0,
519
max_tokens=500,
517
- extra_body=dict(guided_json=TEST_SCHEMA))
520
+ extra_body=dict(guided_json=TEST_SCHEMA,
521
+ guided_decoding_backend=guided_decoding_backend))
522
523
assert completion.id is not None
524
assert completion.choices is not None and len(completion.choices) == 3
@@ -524,7 +528,10 @@ async def test_guided_json_completion(server, client: openai.AsyncOpenAI):
528
jsonschema.validate(instance=output_json, schema=TEST_SCHEMA)
529
530
527
-async def test_guided_json_chat(server, client: openai.AsyncOpenAI):
531
+@pytest.mark.parametrize("guided_decoding_backend",
532
+ ["outlines", "lm-format-enforcer"])
533
+async def test_guided_json_chat(server, client: openai.AsyncOpenAI,
534
+ guided_decoding_backend: str):
535
messages = [{
536
"role": "system",
537
"content": "you are a helpful assistant"
@@ -538,8 +545,9 @@ async def test_guided_json_chat(server, client: openai.AsyncOpenAI):
545
chat_completion = await client.chat.completions.create(
546
model=MODEL_NAME,
547
messages=messages,
541
- max_tokens=500,
542
- extra_body=dict(guided_json=TEST_SCHEMA))
548
+ max_tokens=1000,
549
+ extra_body=dict(guided_json=TEST_SCHEMA,
550
+ guided_decoding_backend=guided_decoding_backend))
551
message = chat_completion.choices[0].message
552
assert message.content is not None
553
json1 = json.loads(message.content)
@@ -555,8 +563,9 @@ async def test_guided_json_chat(server, client: openai.AsyncOpenAI):
563
chat_completion = await client.chat.completions.create(
564
model=MODEL_NAME,
565
messages=messages,
558
- max_tokens=500,
559
- extra_body=dict(guided_json=TEST_SCHEMA))
566
+ max_tokens=1000,
567
+ extra_body=dict(guided_json=TEST_SCHEMA,
568
+ guided_decoding_backend=guided_decoding_backend))
569
message = chat_completion.choices[0].message
570
assert message.content is not None
571
json2 = json.loads(message.content)
@@ -565,14 +574,18 @@ async def test_guided_json_chat(server, client: openai.AsyncOpenAI):
574
assert json1["age"] != json2["age"]
575
576
568
-async def test_guided_regex_completion(server, client: openai.AsyncOpenAI):
577
+@pytest.mark.parametrize("guided_decoding_backend",
578
+ ["outlines", "lm-format-enforcer"])
579
+async def test_guided_regex_completion(server, client: openai.AsyncOpenAI,
580
+ guided_decoding_backend: str):
581
completion = await client.completions.create(
582
model=MODEL_NAME,
583
prompt=f"Give an example IPv4 address with this regex: {TEST_REGEX}",
584
n=3,
585
temperature=1.0,
586
max_tokens=20,
575
- extra_body=dict(guided_regex=TEST_REGEX))
587
+ extra_body=dict(guided_regex=TEST_REGEX,
588
+ guided_decoding_backend=guided_decoding_backend))
589
590
assert completion.id is not None
591
assert completion.choices is not None and len(completion.choices) == 3
@@ -581,7 +594,10 @@ async def test_guided_regex_completion(server, client: openai.AsyncOpenAI):
594
assert re.fullmatch(TEST_REGEX, completion.choices[i].text) is not None
595
596
584
-async def test_guided_regex_chat(server, client: openai.AsyncOpenAI):
597
+@pytest.mark.parametrize("guided_decoding_backend",
598
+ ["outlines", "lm-format-enforcer"])
599
+async def test_guided_regex_chat(server, client: openai.AsyncOpenAI,
600
+ guided_decoding_backend: str):
601
messages = [{
602
"role": "system",
603
"content": "you are a helpful assistant"
@@ -595,7 +611,8 @@ async def test_guided_regex_chat(server, client: openai.AsyncOpenAI):
611
model=MODEL_NAME,
612
messages=messages,
613
max_tokens=20,
598
- extra_body=dict(guided_regex=TEST_REGEX))
614
+ extra_body=dict(guided_regex=TEST_REGEX,
615
+ guided_decoding_backend=guided_decoding_backend))
616
ip1 = chat_completion.choices[0].message.content
617
assert ip1 is not None
618
assert re.fullmatch(TEST_REGEX, ip1) is not None
@@ -606,21 +623,26 @@ async def test_guided_regex_chat(server, client: openai.AsyncOpenAI):
623
model=MODEL_NAME,
624
messages=messages,
625
max_tokens=20,
609
- extra_body=dict(guided_regex=TEST_REGEX))
626
+ extra_body=dict(guided_regex=TEST_REGEX,
627
+ guided_decoding_backend=guided_decoding_backend))
628
ip2 = chat_completion.choices[0].message.content
629
assert ip2 is not None
630
assert re.fullmatch(TEST_REGEX, ip2) is not None
631
assert ip1 != ip2
632
633
616
-async def test_guided_choice_completion(server, client: openai.AsyncOpenAI):
634
+@pytest.mark.parametrize("guided_decoding_backend",
635
+ ["outlines", "lm-format-enforcer"])
636
+async def test_guided_choice_completion(server, client: openai.AsyncOpenAI,
637
+ guided_decoding_backend: str):
638
completion = await client.completions.create(
639
model=MODEL_NAME,
640
prompt="The best language for type-safe systems programming is ",
641
n=2,
642
temperature=1.0,
643
max_tokens=10,
623
- extra_body=dict(guided_choice=TEST_CHOICE))
644
+ extra_body=dict(guided_choice=TEST_CHOICE,
645
+ guided_decoding_backend=guided_decoding_backend))
646
647
assert completion.id is not None
648
assert completion.choices is not None and len(completion.choices) == 2
@@ -628,7 +650,10 @@ async def test_guided_choice_completion(server, client: openai.AsyncOpenAI):
650
assert completion.choices[i].text in TEST_CHOICE
651
652
631
-async def test_guided_choice_chat(server, client: openai.AsyncOpenAI):
653
+@pytest.mark.parametrize("guided_decoding_backend",
654
+ ["outlines", "lm-format-enforcer"])
655
+async def test_guided_choice_chat(server, client: openai.AsyncOpenAI,
656
+ guided_decoding_backend: str):
657
messages = [{
658
"role": "system",
659
"content": "you are a helpful assistant"
@@ -642,7 +667,8 @@ async def test_guided_choice_chat(server, client: openai.AsyncOpenAI):
667
model=MODEL_NAME,
668
messages=messages,
669
max_tokens=10,
645
- extra_body=dict(guided_choice=TEST_CHOICE))
670
+ extra_body=dict(guided_choice=TEST_CHOICE,
671
+ guided_decoding_backend=guided_decoding_backend))
672
choice1 = chat_completion.choices[0].message.content
673
assert choice1 in TEST_CHOICE
674
@@ -655,18 +681,23 @@ async def test_guided_choice_chat(server, client: openai.AsyncOpenAI):
681
model=MODEL_NAME,
682
messages=messages,
683
max_tokens=10,
658
- extra_body=dict(guided_choice=TEST_CHOICE))
684
+ extra_body=dict(guided_choice=TEST_CHOICE,
685
+ guided_decoding_backend=guided_decoding_backend))
686
choice2 = chat_completion.choices[0].message.content
687
assert choice2 in TEST_CHOICE
688
assert choice1 != choice2
689
690
664
-async def test_guided_decoding_type_error(server, client: openai.AsyncOpenAI):
691
+@pytest.mark.parametrize("guided_decoding_backend",
692
+ ["outlines", "lm-format-enforcer"])
693
+async def test_guided_decoding_type_error(server, client: openai.AsyncOpenAI,
694
+ guided_decoding_backend: str):
695
with pytest.raises(openai.BadRequestError):
696
_ = await client.completions.create(
697
model=MODEL_NAME,
698
prompt="Give an example JSON that fits this schema: 42",
669
- extra_body=dict(guided_json=42))
699
+ extra_body=dict(guided_json=42,
700
+ guided_decoding_backend=guided_decoding_backend))
701
702
messages = [{
703
"role": "system",
vllm/config.py
+21
-5
@@ -66,8 +66,8 @@ class ModelConfig:
66
weights. If None, we assume the model weights are not quantized.
67
quantization_param_path: Path to JSON file containing scaling factors.
68
Used to load KV cache scaling factors into the model when KV cache
69
- type is FP8_E4M3 on ROCm (AMD GPU). In the future these will also
70
- be used to load activation and weight scaling factors when the
69
+ type is FP8_E4M3 on ROCm (AMD GPU). In the future these will also
70
+ be used to load activation and weight scaling factors when the
71
model dtype is FP8_E4M3 on ROCm.
72
enforce_eager: Whether to enforce eager execution. If True, we will
73
disable CUDA graph and always execute the model in eager mode.
@@ -422,7 +422,7 @@ class CacheConfig:
422
@dataclass
423
class TokenizerPoolConfig:
424
"""Configuration for the tokenizer pool.
425
-
425
+
426
Args:
427
pool_size: Number of tokenizer workers in the pool.
428
pool_type: Type of the pool.
@@ -446,9 +446,9 @@ class TokenizerPoolConfig:
446
tokenizer_pool_extra_config: Optional[Union[str, dict]]
447
) -> Optional["TokenizerPoolConfig"]:
448
"""Create a TokenizerPoolConfig from the given parameters.
449
-
449
+
450
If tokenizer_pool_size is 0, return None.
451
-
451
+
452
Args:
453
tokenizer_pool_size: Number of tokenizer workers in the pool.
454
tokenizer_pool_type: Type of the pool.
@@ -1079,6 +1079,21 @@ def _get_and_verify_max_len(
1079
return int(max_model_len)
1080
1081
1082
+@dataclass
1083
+class DecodingConfig:
1084
+ """Dataclass which contains the decoding strategy of the engine"""
1085
+
1086
+ # Which guided decoding algo to use. 'outlines' / 'lm-format-enforcer'
1087
+ guided_decoding_backend: str = 'outlines'
1088
+
1089
+ def __post_init__(self):
1090
+ valid_guided_backends = ['outlines', 'lm-format-enforcer']
1091
+ backend = self.guided_decoding_backend
1092
+ if backend not in valid_guided_backends:
1093
+ raise ValueError(f"Invalid guided_decoding_backend '{backend},"
1094
+ f"must be one of {valid_guided_backends}")
1095
+
1096
+
1097
@dataclass(frozen=True)
1098
class EngineConfig:
1099
"""Dataclass which contains all engine-related configuration. This
@@ -1093,6 +1108,7 @@ class EngineConfig:
1108
lora_config: Optional[LoRAConfig]
1109
vision_language_config: Optional[VisionLanguageConfig]
1110
speculative_config: Optional[SpeculativeConfig]
1111
+ decoding_config: Optional[DecodingConfig]
1112
tensorizer_config: Optional[TensorizerConfig]
1113
1114
def __post_init__(self):
vllm/engine/arg_utils.py
+15
-3
@@ -5,9 +5,9 @@ import os
5
from dataclasses import dataclass
6
from typing import BinaryIO, Optional, Union
7
8
-from vllm.config import (CacheConfig, DeviceConfig, EngineConfig, LoRAConfig,
9
- ModelConfig, ParallelConfig, SchedulerConfig,
10
- SpeculativeConfig, TensorizerConfig,
8
+from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig,
9
+ EngineConfig, LoRAConfig, ModelConfig, ParallelConfig,
10
+ SchedulerConfig, SpeculativeConfig, TensorizerConfig,
11
TokenizerPoolConfig, VisionLanguageConfig)
12
from vllm.model_executor.tensorizer_loader import TensorizerArgs
13
from vllm.utils import str_to_int_tuple
@@ -80,6 +80,7 @@ class EngineArgs:
80
scheduler_delay_factor: float = 0.0
81
enable_chunked_prefill: bool = False
82
83
+ guided_decoding_backend: str = 'outlines'
84
# Speculative decoding configuration.
85
speculative_model: Optional[str] = None
86
num_speculative_tokens: Optional[int] = None
@@ -200,6 +201,13 @@ class EngineArgs:
201
default=EngineArgs.max_model_len,
202
help='model context length. If unspecified, '
203
'will be automatically derived from the model.')
204
+ parser.add_argument(
205
+ '--guided-decoding-backend',
206
+ type=str,
207
+ default='outlines',
208
+ choices=['outlines', 'lm-format-enforcer'],
209
+ help='Which engine will be used for guided decoding'
210
+ ' (JSON schema / regex etc)')
211
# Parallel arguments
212
parser.add_argument('--worker-use-ray',
213
action='store_true',
@@ -511,6 +519,9 @@ class EngineArgs:
519
else:
520
vision_language_config = None
521
522
+ decoding_config = DecodingConfig(
523
+ guided_decoding_backend=self.guided_decoding_backend)
524
+
525
return EngineConfig(model_config=model_config,
526
cache_config=cache_config,
527
parallel_config=parallel_config,
@@ -519,6 +530,7 @@ class EngineArgs:
530
lora_config=lora_config,
531
vision_language_config=vision_language_config,
532
speculative_config=speculative_config,
533
+ decoding_config=decoding_config,
534
tensorizer_config=tensorizer_config)
535
536
vllm/engine/llm_engine.py
+7
-3
@@ -4,9 +4,10 @@ from typing import Iterable, List, Optional, Tuple, Type, Union
4
from transformers import PreTrainedTokenizer
5
6
import vllm
7
-from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
8
- ParallelConfig, SchedulerConfig, SpeculativeConfig,
9
- TensorizerConfig, VisionLanguageConfig)
7
+from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig, LoRAConfig,
8
+ ModelConfig, ParallelConfig, SchedulerConfig,
9
+ SpeculativeConfig, TensorizerConfig,
10
+ VisionLanguageConfig)
11
from vllm.core.scheduler import Scheduler, SchedulerOutputs
12
from vllm.engine.arg_utils import EngineArgs
13
from vllm.engine.metrics import StatLogger, Stats
@@ -74,6 +75,7 @@ class LLMEngine:
75
lora_config: Optional[LoRAConfig],
76
vision_language_config: Optional[VisionLanguageConfig],
77
speculative_config: Optional[SpeculativeConfig],
78
+ decoding_config: Optional[DecodingConfig],
79
tensorizer_config: Optional[TensorizerConfig],
80
executor_class: Type[ExecutorBase],
81
log_stats: bool,
@@ -100,6 +102,7 @@ class LLMEngine:
102
f"kv_cache_dtype={cache_config.cache_dtype}, "
103
f"quantization_param_path={model_config.quantization_param_path}, "
104
f"device_config={device_config.device}, "
105
+ f"decoding_config={decoding_config!r}, "
106
f"seed={model_config.seed})")
107
# TODO(woosuk): Print more configs in debug mode.
108
@@ -111,6 +114,7 @@ class LLMEngine:
114
self.scheduler_config = scheduler_config
115
self.device_config = device_config
116
self.speculative_config = speculative_config
117
+ self.decoding_config = decoding_config or DecodingConfig()
118
self.tensorizer_config = tensorizer_config
119
self.log_stats = log_stats
120
vllm/entrypoints/openai/protocol.py
+12
@@ -133,6 +133,12 @@ class ChatCompletionRequest(BaseModel):
133
description=(
134
"If specified, the output will follow the context free grammar."),
135
)
136
+ guided_decoding_backend: Optional[str] = Field(
137
+ default=None,
138
+ description=(
139
+ "If specified, will override the default guided decoding backend "
140
+ "of the server for this specific request. If set, must be either "
141
+ "'outlines' / 'lm-format-enforcer'"))
142
143
# doc: end-chat-completion-extra-params
144
@@ -265,6 +271,12 @@ class CompletionRequest(BaseModel):
271
description=(
272
"If specified, the output will follow the context free grammar."),
273
)
274
+ guided_decoding_backend: Optional[str] = Field(
275
+ default=None,
276
+ description=(
277
+ "If specified, will override the default guided decoding backend "
278
+ "of the server for this specific request. If set, must be one of "
279
+ "'outlines' / 'lm-format-enforcer'"))
280
281
# doc: end-completion-extra-params
282
vllm/entrypoints/openai/serving_chat.py
+5
-1
@@ -68,9 +68,13 @@ class OpenAIServingChat(OpenAIServing):
68
request, prompt=prompt)
69
sampling_params = request.to_sampling_params()
70
lora_request = self._maybe_get_lora(request)
71
+ decoding_config = self.engine.engine.decoding_config
72
+ guided_decoding_backend = request.guided_decoding_backend \
73
+ or decoding_config.guided_decoding_backend
74
guided_decode_logits_processor = (
75
await get_guided_decoding_logits_processor(
73
- request, await self.engine.get_tokenizer()))
76
+ guided_decoding_backend, request, await
77
+ self.engine.get_tokenizer()))
78
if guided_decode_logits_processor:
79
if sampling_params.logits_processors is None:
80
sampling_params.logits_processors = []
vllm/entrypoints/openai/serving_completion.py
+5
-1
@@ -88,9 +88,13 @@ class OpenAIServingCompletion(OpenAIServing):
88
try:
89
sampling_params = request.to_sampling_params()
90
lora_request = self._maybe_get_lora(request)
91
+ decoding_config = self.engine.engine.decoding_config
92
+ guided_decoding_backend = request.guided_decoding_backend \
93
+ or decoding_config.guided_decoding_backend
94
guided_decode_logit_processor = (
95
await get_guided_decoding_logits_processor(
93
- request, await self.engine.get_tokenizer()))
96
+ guided_decoding_backend, request, await
97
+ self.engine.get_tokenizer()))
98
if guided_decode_logit_processor is not None:
99
if sampling_params.logits_processors is None:
100
sampling_params.logits_processors = []
vllm/model_executor/guided_decoding/__init__.py
new
+25
@@ -0,0 +1,25 @@
1
+from typing import Optional, Union
2
+
3
+from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
4
+ CompletionRequest)
5
+from vllm.model_executor.guided_decoding.lm_format_enforcer_decoding import (
6
+ get_lm_format_enforcer_guided_decoding_logits_processor)
7
+from vllm.model_executor.guided_decoding.outlines_decoding import (
8
+ get_outlines_guided_decoding_logits_processor)
9
+from vllm.sampling_params import LogitsProcessor
10
+
11
+
12
+async def get_guided_decoding_logits_processor(
13
+ guided_decoding_backend: str, request: Union[CompletionRequest,
14
+ ChatCompletionRequest],
15
+ tokenizer) -> Optional[LogitsProcessor]:
16
+ if guided_decoding_backend == 'outlines':
17
+ return await get_outlines_guided_decoding_logits_processor(
18
+ request, tokenizer)
19
+ if guided_decoding_backend == 'lm-format-enforcer':
20
+ return await get_lm_format_enforcer_guided_decoding_logits_processor(
21
+ request, tokenizer)
22
+
23
+ raise ValueError(
24
+ f"Unknown guided decoding backend '{guided_decoding_backend}'. "
25
+ "Must be one of 'outlines, 'lm-format-enforcer'")
vllm/model_executor/guided_decoding/lm_format_enforcer_decoding.py
new
+69
@@ -0,0 +1,69 @@
1
+from functools import lru_cache
2
+from json import loads as json_loads
3
+from typing import Optional, Union
4
+
5
+from lmformatenforcer import (CharacterLevelParser, JsonSchemaParser,
6
+ RegexParser, StringParser,
7
+ TokenEnforcerTokenizerData, UnionParser)
8
+from lmformatenforcer.integrations.vllm import (
9
+ build_vllm_logits_processor, build_vllm_token_enforcer_tokenizer_data)
10
+from pydantic import BaseModel
11
+from transformers import PreTrainedTokenizerBase
12
+
13
+from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
14
+ CompletionRequest)
15
+from vllm.model_executor.guided_decoding.outlines_decoding import (
16
+ get_outlines_guided_decoding_logits_processor)
17
+from vllm.sampling_params import LogitsProcessor
18
+
19
+
20
+async def get_lm_format_enforcer_guided_decoding_logits_processor(
21
+ request: Union[CompletionRequest, ChatCompletionRequest],
22
+ tokenizer) -> Optional[LogitsProcessor]:
23
+ """
24
+ Given an OpenAI-compatible request, check for guided decoding parameters
25
+ and get the necessary logits processor for the given guide.
26
+ We cache logit processors by (guide, tokenizer), and on cache hit
27
+ we make a shallow copy to reuse the same underlying FSM.
28
+ """
29
+
30
+ tokenizer_data = _cached_build_vllm_token_enforcer_tokenizer_data(
31
+ tokenizer)
32
+ character_level_parser: CharacterLevelParser
33
+ if request.guided_json:
34
+ schema = _normalize_json_schema_object(request.guided_json)
35
+ character_level_parser = JsonSchemaParser(schema)
36
+ elif request.guided_choice:
37
+ character_level_parser = UnionParser(
38
+ [StringParser(choice) for choice in request.guided_choice])
39
+ elif request.guided_regex:
40
+ character_level_parser = RegexParser(request.guided_regex)
41
+ elif request.guided_grammar:
42
+ # CFG grammar not supported by LMFE, revert to outlines
43
+ return await get_outlines_guided_decoding_logits_processor(
44
+ request, tokenizer)
45
+ elif (request.response_format is not None
46
+ and request.response_format.type == "json_object"):
47
+ character_level_parser = JsonSchemaParser(
48
+ None) # None means any json object
49
+ else:
50
+ return None
51
+
52
+ logits_processor = build_vllm_logits_processor(tokenizer_data,
53
+ character_level_parser)
54
+ return logits_processor
55
+
56
+
57
+def _normalize_json_schema_object(schema: Union[str, dict, BaseModel]) -> dict:
58
+ if isinstance(schema, str):
59
+ return json_loads(schema)
60
+ if isinstance(schema, dict):
61
+ return schema
62
+ if isinstance(schema, BaseModel):
63
+ return schema.model_json_schema()
64
+
65
+
66
+@lru_cache
67
+def _cached_build_vllm_token_enforcer_tokenizer_data(
68
+ tokenizer: PreTrainedTokenizerBase) -> TokenEnforcerTokenizerData:
69
+ return build_vllm_token_enforcer_tokenizer_data(tokenizer)
vllm/model_executor/guided_decoding/outlines_decoding.py
renamed
+3
-4
@@ -12,9 +12,8 @@ from transformers import PreTrainedTokenizerBase
12
13
from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
14
CompletionRequest)
15
-from vllm.model_executor.guided_logits_processors import (CFGLogitsProcessor,
16
- JSONLogitsProcessor,
17
- RegexLogitsProcessor)
15
+from vllm.model_executor.guided_decoding.outlines_logits_processors import (
16
+ CFGLogitsProcessor, JSONLogitsProcessor, RegexLogitsProcessor)
17
18
19
class GuidedDecodingMode(Enum):
@@ -54,7 +53,7 @@ pair : UNESCAPED_STRING ":" value
53
global_thread_pool = None # used for generating logits processor fsm
54
55
57
-async def get_guided_decoding_logits_processor(
56
+async def get_outlines_guided_decoding_logits_processor(
57
request: Union[CompletionRequest, ChatCompletionRequest],
58
tokenizer) -> Union[JSONLogitsProcessor, RegexLogitsProcessor]:
59
"""
vllm/model_executor/guided_decoding/outlines_logits_processors.py
renamed
+52
-48
@@ -13,9 +13,11 @@
13
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
# See the License for the specific language governing permissions and
15
# limitations under the License.
16
+import copy
17
import json
18
import math
19
from collections import defaultdict
20
+from functools import lru_cache
21
from typing import Callable, DefaultDict, Dict, List, Optional, Union
22
23
import torch
@@ -27,50 +29,6 @@ from transformers import PreTrainedTokenizerBase
29
30
class BaseLogitsProcessor:
31
30
- def adapt_tokenizer(self, tokenizer: PreTrainedTokenizerBase):
31
- """Adapt vLLM's tokenizer to use to compile the FSM.
32
-
33
- The API of Outlines tokenizers is slightly different to that of
34
- `transformers`. The decoder of outlines, returns a list whereas
35
- the decode of vLLM returns an str. To sync the vLLM decoder with
36
- outlines internal api, the decoder should be adapted. In addition
37
- we need to handle the missing spaces to Llama's tokenizer to be
38
- able to compile FSMs for this model.
39
-
40
- """
41
- if getattr(tokenizer, "_outlines_adapted", False):
42
- return tokenizer
43
-
44
- tokenizer.vocabulary = tokenizer.get_vocab()
45
- tokenizer.special_tokens = set(tokenizer.all_special_tokens)
46
-
47
- def convert_token_to_string(token: str) -> str:
48
- from transformers.file_utils import SPIECE_UNDERLINE
49
-
50
- string = tokenizer.convert_tokens_to_string([token])
51
-
52
- # A hack to handle missing spaces to HF's Llama tokenizers
53
- if token.startswith(SPIECE_UNDERLINE) or token == "<0x20>":
54
- return " " + string
55
-
56
- return string
57
-
58
- def change_decoder(
59
- decoder: Callable[[List[int]], str]
60
- ) -> Callable[[List[int]], List[str]]:
61
- """Sync vLLM's decoder with the outlines by returning list."""
62
-
63
- def new_decoder(inp_tokens: List[int]) -> List[str]:
64
- return [decoder(inp_tokens)]
65
-
66
- return new_decoder
67
-
68
- tokenizer.convert_token_to_string = convert_token_to_string
69
- tokenizer.decode = change_decoder(tokenizer.decode)
70
- setattr(tokenizer, "_outlines_adapted", True) # noqa: B010
71
-
72
- return tokenizer
73
-
32
def init_state(self):
33
"""Initialize the FSM states."""
34
self.fsm_state: DefaultDict[int, int] = defaultdict(int)
@@ -78,7 +36,6 @@ class BaseLogitsProcessor:
36
def __call__(self, input_ids: List[int],
37
scores: torch.Tensor) -> torch.Tensor:
38
"""Use the FSM to bias the logits before sampling the next token."""
81
-
39
seq_id = hash(tuple(input_ids))
40
41
if len(input_ids) == 0:
@@ -96,7 +53,6 @@ class BaseLogitsProcessor:
53
device=scores.device)
54
mask[allowed_tokens] = 0
55
scores.add_(mask)
99
-
56
return scores
57
58
@@ -113,7 +69,7 @@ class RegexLogitsProcessor(BaseLogitsProcessor):
69
The model's tokenizer
70
71
"""
116
- tokenizer = self.adapt_tokenizer(tokenizer)
72
+ tokenizer = _adapt_tokenizer(tokenizer)
73
fsm = RegexFSM(regex_string, tokenizer)
74
self.fsm = fsm
75
@@ -167,6 +123,54 @@ class CFGLogitsProcessor(BaseLogitsProcessor):
123
The model's tokenizer
124
125
"""
170
- tokenizer = self.adapt_tokenizer(tokenizer)
126
+ tokenizer = _adapt_tokenizer(tokenizer)
127
fsm = CFGFSM(cfg, tokenizer)
128
self.fsm = fsm
129
+
130
+
131
+@lru_cache
132
+def _adapt_tokenizer(tokenizer: PreTrainedTokenizerBase):
133
+ """Adapt vLLM's tokenizer to use to compile the FSM.
134
+
135
+ The API of Outlines tokenizers is slightly different to that of
136
+ `transformers`. The decoder of outlines, returns a list whereas
137
+ the decode of vLLM returns an str. To sync the vLLM decoder with
138
+ outlines internal api, the decoder should be adapted. In addition
139
+ we need to handle the missing spaces to Llama's tokenizer to be
140
+ able to compile FSMs for this model.
141
+
142
+ """
143
+ if getattr(tokenizer, "_outlines_adapted", False):
144
+ return tokenizer
145
+
146
+ tokenizer = copy.deepcopy(tokenizer)
147
+
148
+ tokenizer.vocabulary = tokenizer.get_vocab()
149
+ tokenizer.special_tokens = set(tokenizer.all_special_tokens)
150
+
151
+ def convert_token_to_string(token: str) -> str:
152
+ from transformers.file_utils import SPIECE_UNDERLINE
153
+
154
+ string = tokenizer.convert_tokens_to_string([token])
155
+
156
+ # A hack to handle missing spaces to HF's Llama tokenizers
157
+ if token.startswith(SPIECE_UNDERLINE) or token == "<0x20>":
158
+ return " " + string
159
+
160
+ return string
161
+
162
+ def change_decoder(
163
+ decoder: Callable[[List[int]],
164
+ str]) -> Callable[[List[int]], List[str]]:
165
+ """Sync vLLM's decoder with the outlines by returning list."""
166
+
167
+ def new_decoder(inp_tokens: List[int]) -> List[str]:
168
+ return [decoder(inp_tokens)]
169
+
170
+ return new_decoder
171
+
172
+ tokenizer.convert_token_to_string = convert_token_to_string
173
+ tokenizer.decode = change_decoder(tokenizer.decode)
174
+ setattr(tokenizer, "_outlines_adapted", True) # noqa: B010
175
+
176
+ return tokenizer