A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
Allow user to define whitespace pattern for outlines (#4305)
Robert Caulk committed
May 1, 2024 at 05:48 UTC
c3845d82dc3d1831714898114f87d9c103e2dd41
4 files changed
+21
-8
tests/entrypoints/test_guided_processors.py
+3
-1
@@ -57,7 +57,9 @@ def test_guided_logits_processors():
57
"""Basic unit test for RegexLogitsProcessor and JSONLogitsProcessor."""
58
tokenizer = AutoTokenizer.from_pretrained('HuggingFaceH4/zephyr-7b-beta')
59
regex_LP = RegexLogitsProcessor(TEST_REGEX, tokenizer)
60
- json_LP = JSONLogitsProcessor(TEST_SCHEMA, tokenizer)
60
+ json_LP = JSONLogitsProcessor(TEST_SCHEMA,
61
+ tokenizer,
62
+ whitespace_pattern=None)
63
64
regex_LP.init_state()
65
token_ids = tokenizer.encode(
vllm/entrypoints/openai/protocol.py
+10
@@ -146,6 +146,11 @@ class ChatCompletionRequest(OpenAIBaseModel):
146
"If specified, will override the default guided decoding backend "
147
"of the server for this specific request. If set, must be either "
148
"'outlines' / 'lm-format-enforcer'"))
149
+ guided_whitespace_pattern: Optional[str] = Field(
150
+ default=None,
151
+ description=(
152
+ "If specified, will override the default whitespace pattern "
153
+ "for guided json decoding."))
154
155
# doc: end-chat-completion-extra-params
156
@@ -285,6 +290,11 @@ class CompletionRequest(OpenAIBaseModel):
290
"If specified, will override the default guided decoding backend "
291
"of the server for this specific request. If set, must be one of "
292
"'outlines' / 'lm-format-enforcer'"))
293
+ guided_whitespace_pattern: Optional[str] = Field(
294
+ default=None,
295
+ description=(
296
+ "If specified, will override the default whitespace pattern "
297
+ "for guided json decoding."))
298
299
# doc: end-completion-extra-params
300
vllm/model_executor/guided_decoding/outlines_decoding.py
+5
-3
@@ -74,7 +74,8 @@ async def get_outlines_guided_decoding_logits_processor(
74
75
result = await loop.run_in_executor(global_thread_pool,
76
_get_cached_logits_processor, guide,
77
- tokenizer, mode)
77
+ tokenizer, mode,
78
+ request.guided_whitespace_pattern)
79
80
logits_processor = copy(result)
81
# reset logits processor's internal state
@@ -117,9 +118,10 @@ def _get_guide_and_mode(
118
@lru_cache(maxsize=32)
119
def _get_cached_logits_processor(guide: str,
120
tokenizer: PreTrainedTokenizerBase,
120
- mode: GuidedDecodingMode):
121
+ mode: GuidedDecodingMode,
122
+ whitespace_pattern: Union[str, None]):
123
if mode == GuidedDecodingMode.JSON:
122
- return JSONLogitsProcessor(guide, tokenizer)
124
+ return JSONLogitsProcessor(guide, tokenizer, whitespace_pattern)
125
elif mode == GuidedDecodingMode.REGEX or mode == GuidedDecodingMode.CHOICE:
126
return RegexLogitsProcessor(guide, tokenizer)
127
elif mode == GuidedDecodingMode.GRAMMAR:
vllm/model_executor/guided_decoding/outlines_logits_processors.py
+3
-4
@@ -18,7 +18,7 @@ import json
18
import math
19
from collections import defaultdict
20
from functools import lru_cache
21
-from typing import Callable, DefaultDict, Dict, List, Optional, Union
21
+from typing import Callable, DefaultDict, Dict, List, Union
22
23
import torch
24
from outlines.fsm.fsm import CFGFSM, FSM, RegexFSM
@@ -80,10 +80,9 @@ class RegexLogitsProcessor(BaseLogitsProcessor):
80
81
class JSONLogitsProcessor(RegexLogitsProcessor):
82
83
- def __init__(self,
84
- schema: Union[str, Dict, BaseModel],
83
+ def __init__(self, schema: Union[str, Dict, BaseModel],
84
tokenizer: PreTrainedTokenizerBase,
86
- whitespace_pattern: Optional[str] = None):
85
+ whitespace_pattern: Union[str, None]):
86
"""Compile the FSM that drives the JSON-guided generation.
87
88
Parameters