A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Frontend][Core] Update Outlines Integration from `FSM` to `Guide` (#4109)
Co-authored-by: Simon Mo <simon.mo@hey.com> Co-authored-by: Breno Faria <breno.faria@intrafind.com>
Breno Faria committed
Jun 6, 2024 at 01:49 UTC
7b0a0dfb22907505441f8a4a5eb882cbca4d2acf
4 files changed
+49
-48
requirements-common.txt
+1
-1
@@ -17,6 +17,6 @@ prometheus_client >= 0.18.0
17
prometheus-fastapi-instrumentator >= 7.0.0
18
tiktoken >= 0.6.0 # Required for DBRX tokenizer
19
lm-format-enforcer == 0.10.1
20
-outlines == 0.0.34 # Requires torch >= 2.1.0
20
+outlines >= 0.0.43 # Requires torch >= 2.1.0
21
typing_extensions
22
filelock >= 3.10.4 # filelock starts to support `mode` argument from 3.10.4
tests/entrypoints/test_guided_processors.py
-2
@@ -63,7 +63,6 @@ def test_guided_logits_processors():
63
tokenizer,
64
whitespace_pattern=None)
65
66
- regex_LP.init_state()
66
token_ids = tokenizer.encode(
67
f"Give an example IPv4 address with this regex: {TEST_REGEX}")
68
tensor = torch.rand(32000)
@@ -72,7 +71,6 @@ def test_guided_logits_processors():
71
assert tensor.shape == original_tensor.shape
72
assert not torch.allclose(tensor, original_tensor)
73
75
- json_LP.init_state()
74
token_ids = tokenizer.encode(
75
f"Give an employee profile that fits this schema: {TEST_SCHEMA}")
76
tensor = torch.rand(32000)
vllm/model_executor/guided_decoding/outlines_decoding.py
+12
-19
@@ -1,8 +1,6 @@
1
import asyncio
2
import concurrent.futures
3
-from copy import copy
3
from enum import Enum
5
-from functools import lru_cache
4
from json import dumps as json_dumps
5
from re import escape as regex_escape
6
from typing import Tuple, Union
@@ -54,8 +52,10 @@ global_thread_pool = None # used for generating logits processor fsm
52
53
54
async def get_outlines_guided_decoding_logits_processor(
57
- request: Union[CompletionRequest, ChatCompletionRequest],
58
- tokenizer) -> Union[JSONLogitsProcessor, RegexLogitsProcessor, None]:
55
+ request: Union[CompletionRequest,
56
+ ChatCompletionRequest], tokenizer: PreTrainedTokenizerBase
57
+) -> Union[JSONLogitsProcessor, RegexLogitsProcessor, CFGLogitsProcessor,
58
+ None]:
59
"""
60
Given an OpenAI-compatible request, check for guided decoding parameters
61
and get the necessary logits processor for the given guide.
@@ -64,7 +64,7 @@ async def get_outlines_guided_decoding_logits_processor(
64
"""
65
global global_thread_pool
66
guide, mode = _get_guide_and_mode(request)
67
- if not guide:
67
+ if not guide or not mode:
68
return None
69
70
if global_thread_pool is None:
@@ -72,15 +72,9 @@ async def get_outlines_guided_decoding_logits_processor(
72
max_workers=2)
73
loop = asyncio.get_running_loop()
74
75
- result = await loop.run_in_executor(global_thread_pool,
76
- _get_cached_logits_processor, guide,
77
- tokenizer, mode,
78
- request.guided_whitespace_pattern)
79
-
80
- logits_processor = copy(result)
81
- # reset logits processor's internal state
82
- logits_processor.init_state()
83
- return logits_processor
75
+ return await loop.run_in_executor(global_thread_pool,
76
+ _get_logits_processor, guide, tokenizer,
77
+ mode, request.guided_whitespace_pattern)
78
79
80
def _get_guide_and_mode(
@@ -115,11 +109,10 @@ def _get_guide_and_mode(
109
return None, None
110
111
118
-@lru_cache(maxsize=32)
119
-def _get_cached_logits_processor(guide: str,
120
- tokenizer: PreTrainedTokenizerBase,
121
- mode: GuidedDecodingMode,
122
- whitespace_pattern: Union[str, None]):
112
+def _get_logits_processor(
113
+ guide: str, tokenizer: PreTrainedTokenizerBase, mode: GuidedDecodingMode,
114
+ whitespace_pattern: Union[str, None]
115
+) -> Union[JSONLogitsProcessor, RegexLogitsProcessor, CFGLogitsProcessor]:
116
if mode == GuidedDecodingMode.JSON:
117
return JSONLogitsProcessor(guide, tokenizer, whitespace_pattern)
118
elif mode == GuidedDecodingMode.REGEX or mode == GuidedDecodingMode.CHOICE:
vllm/model_executor/guided_decoding/outlines_logits_processors.py
+36
-26
@@ -21,7 +21,7 @@ from functools import lru_cache
21
from typing import Callable, DefaultDict, Dict, List, Union
22
23
import torch
24
-from outlines.fsm.fsm import CFGFSM, FSM, RegexFSM
24
+from outlines.fsm.guide import CFGGuide, Generate, Guide, RegexGuide, Write
25
from outlines.fsm.json_schema import build_regex_from_schema
26
from pydantic import BaseModel
27
from transformers import PreTrainedTokenizerBase
@@ -29,28 +29,32 @@ from transformers import PreTrainedTokenizerBase
29
30
class BaseLogitsProcessor:
31
32
- def __init__(self):
33
- # Child class should use initialize in their init.
34
- self.fsm: FSM
35
-
36
- def init_state(self):
37
- """Initialize the FSM states."""
38
- self.fsm_state: DefaultDict[int, int] = defaultdict(int)
32
+ def __init__(self, guide: Guide):
33
+ self._guide: Guide = guide
34
+ self._fsm_state: DefaultDict[int, int] = defaultdict(int)
35
36
def __call__(self, input_ids: List[int],
37
scores: torch.Tensor) -> torch.Tensor:
38
"""Use the FSM to bias the logits before sampling the next token."""
39
seq_id = hash(tuple(input_ids))
40
45
- if len(input_ids) == 0:
46
- self.init_state()
47
- else:
41
+ if len(input_ids) > 0:
42
last_token = input_ids[-1]
43
last_seq_id = hash(tuple(input_ids[:-1]))
50
- self.fsm_state[seq_id] = self.fsm.next_state(
51
- self.fsm_state[last_seq_id], last_token)
44
+ self._fsm_state[seq_id] = self._guide.get_next_state(
45
+ state=self._fsm_state[last_seq_id], token_id=last_token)
46
+
47
+ instruction = self._guide.get_next_instruction(
48
+ state=self._fsm_state[seq_id])
49
53
- allowed_tokens = self.fsm.allowed_token_ids(self.fsm_state[seq_id])
50
+ if type(instruction) == Generate:
51
+ allowed_tokens = instruction.tokens
52
+ elif type(instruction) == Write:
53
+ # TODO: support fast forward tokens
54
+ allowed_tokens = [instruction.tokens[0]]
55
+ else:
56
+ raise TypeError(
57
+ f"Unsupported instruction type {type(instruction)}")
58
59
mask = torch.full((scores.shape[-1], ),
60
-math.inf,
@@ -62,6 +66,13 @@ class BaseLogitsProcessor:
66
67
class RegexLogitsProcessor(BaseLogitsProcessor):
68
69
+ @classmethod
70
+ @lru_cache(maxsize=32)
71
+ def _get_guide(cls, regex_string: str,
72
+ tokenizer: PreTrainedTokenizerBase) -> Guide:
73
+ tokenizer = _adapt_tokenizer(tokenizer)
74
+ return RegexGuide(regex_string, tokenizer)
75
+
76
def __init__(self, regex_string: str, tokenizer: PreTrainedTokenizerBase):
77
"""Compile the FSM that drives the regex-structured generation.
78
@@ -73,9 +84,8 @@ class RegexLogitsProcessor(BaseLogitsProcessor):
84
The model's tokenizer
85
86
"""
76
- tokenizer = _adapt_tokenizer(tokenizer)
77
- fsm = RegexFSM(regex_string, tokenizer)
78
- self.fsm = fsm
87
+ super().__init__(
88
+ RegexLogitsProcessor._get_guide(regex_string, tokenizer))
89
90
91
class JSONLogitsProcessor(RegexLogitsProcessor):
@@ -115,6 +125,12 @@ class JSONLogitsProcessor(RegexLogitsProcessor):
125
126
class CFGLogitsProcessor(BaseLogitsProcessor):
127
128
+ @classmethod
129
+ @lru_cache(maxsize=32)
130
+ def _get_guide(cls, cfg: str, tokenizer: PreTrainedTokenizerBase) -> Guide:
131
+ tokenizer = _adapt_tokenizer(tokenizer)
132
+ return CFGGuide(cfg, tokenizer)
133
+
134
def __init__(self, cfg: str, tokenizer: PreTrainedTokenizerBase):
135
"""Compile the FSM that drives the context free grammar generation.
136
@@ -126,17 +142,11 @@ class CFGLogitsProcessor(BaseLogitsProcessor):
142
The model's tokenizer
143
144
"""
129
- tokenizer = _adapt_tokenizer(tokenizer)
130
- fsm = CFGFSM(cfg, tokenizer)
131
- self.fsm = fsm
132
-
133
- def init_state(self):
134
- """Initialize state with a CFGFSM copy."""
135
- super().init_state()
136
- self.fsm = self.fsm.copy()
145
+ super().__init__(CFGLogitsProcessor._get_guide(cfg, tokenizer))
146
+ self._guide = self._guide.copy()
147
148
139
-@lru_cache
149
+@lru_cache(maxsize=32)
150
def _adapt_tokenizer(tokenizer: PreTrainedTokenizerBase):
151
"""Adapt vLLM's tokenizer to use to compile the FSM.
152