@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c3845d82dc
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

Allow user to define whitespace pattern for outlines (#4305)

Robert Caulk committed May 1, 2024 at 05:48 UTC c3845d82dc3d1831714898114f87d9c103e2dd41
4 files changed +21 -8
tests/entrypoints/test_guided_processors.py
+3 -1
@@ -57,7 +57,9 @@ def test_guided_logits_processors():
57 """Basic unit test for RegexLogitsProcessor and JSONLogitsProcessor."""
58 tokenizer = AutoTokenizer.from_pretrained('HuggingFaceH4/zephyr-7b-beta')
59 regex_LP = RegexLogitsProcessor(TEST_REGEX, tokenizer)
60 - json_LP = JSONLogitsProcessor(TEST_SCHEMA, tokenizer)
60 + json_LP = JSONLogitsProcessor(TEST_SCHEMA,
61 + tokenizer,
62 + whitespace_pattern=None)
63
64 regex_LP.init_state()
65 token_ids = tokenizer.encode(
vllm/entrypoints/openai/protocol.py
+10
@@ -146,6 +146,11 @@ class ChatCompletionRequest(OpenAIBaseModel):
146 "If specified, will override the default guided decoding backend "
147 "of the server for this specific request. If set, must be either "
148 "'outlines' / 'lm-format-enforcer'"))
149 + guided_whitespace_pattern: Optional[str] = Field(
150 + default=None,
151 + description=(
152 + "If specified, will override the default whitespace pattern "
153 + "for guided json decoding."))
154
155 # doc: end-chat-completion-extra-params
156
@@ -285,6 +290,11 @@ class CompletionRequest(OpenAIBaseModel):
290 "If specified, will override the default guided decoding backend "
291 "of the server for this specific request. If set, must be one of "
292 "'outlines' / 'lm-format-enforcer'"))
293 + guided_whitespace_pattern: Optional[str] = Field(
294 + default=None,
295 + description=(
296 + "If specified, will override the default whitespace pattern "
297 + "for guided json decoding."))
298
299 # doc: end-completion-extra-params
300
vllm/model_executor/guided_decoding/outlines_decoding.py
+5 -3
@@ -74,7 +74,8 @@ async def get_outlines_guided_decoding_logits_processor(
74
75 result = await loop.run_in_executor(global_thread_pool,
76 _get_cached_logits_processor, guide,
77 - tokenizer, mode)
77 + tokenizer, mode,
78 + request.guided_whitespace_pattern)
79
80 logits_processor = copy(result)
81 # reset logits processor's internal state
@@ -117,9 +118,10 @@ def _get_guide_and_mode(
118 @lru_cache(maxsize=32)
119 def _get_cached_logits_processor(guide: str,
120 tokenizer: PreTrainedTokenizerBase,
120 - mode: GuidedDecodingMode):
121 + mode: GuidedDecodingMode,
122 + whitespace_pattern: Union[str, None]):
123 if mode == GuidedDecodingMode.JSON:
122 - return JSONLogitsProcessor(guide, tokenizer)
124 + return JSONLogitsProcessor(guide, tokenizer, whitespace_pattern)
125 elif mode == GuidedDecodingMode.REGEX or mode == GuidedDecodingMode.CHOICE:
126 return RegexLogitsProcessor(guide, tokenizer)
127 elif mode == GuidedDecodingMode.GRAMMAR:
vllm/model_executor/guided_decoding/outlines_logits_processors.py
+3 -4
@@ -18,7 +18,7 @@ import json
18 import math
19 from collections import defaultdict
20 from functools import lru_cache
21 -from typing import Callable, DefaultDict, Dict, List, Optional, Union
21 +from typing import Callable, DefaultDict, Dict, List, Union
22
23 import torch
24 from outlines.fsm.fsm import CFGFSM, FSM, RegexFSM
@@ -80,10 +80,9 @@ class RegexLogitsProcessor(BaseLogitsProcessor):
80
81 class JSONLogitsProcessor(RegexLogitsProcessor):
82
83 - def __init__(self,
84 - schema: Union[str, Dict, BaseModel],
83 + def __init__(self, schema: Union[str, Dict, BaseModel],
84 tokenizer: PreTrainedTokenizerBase,
86 - whitespace_pattern: Optional[str] = None):
85 + whitespace_pattern: Union[str, None]):
86 """Compile the FSM that drives the JSON-guided generation.
87
88 Parameters