@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7b0a0dfb22
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Frontend][Core] Update Outlines Integration from `FSM` to `Guide` (#4109)

Co-authored-by: Simon Mo <simon.mo@hey.com> Co-authored-by: Breno Faria <breno.faria@intrafind.com>

Breno Faria committed Jun 6, 2024 at 01:49 UTC 7b0a0dfb22907505441f8a4a5eb882cbca4d2acf
4 files changed +49 -48
requirements-common.txt
+1 -1
@@ -17,6 +17,6 @@ prometheus_client >= 0.18.0
17 prometheus-fastapi-instrumentator >= 7.0.0
18 tiktoken >= 0.6.0 # Required for DBRX tokenizer
19 lm-format-enforcer == 0.10.1
20 -outlines == 0.0.34 # Requires torch >= 2.1.0
20 +outlines >= 0.0.43 # Requires torch >= 2.1.0
21 typing_extensions
22 filelock >= 3.10.4 # filelock starts to support `mode` argument from 3.10.4
tests/entrypoints/test_guided_processors.py
-2
@@ -63,7 +63,6 @@ def test_guided_logits_processors():
63 tokenizer,
64 whitespace_pattern=None)
65
66 - regex_LP.init_state()
66 token_ids = tokenizer.encode(
67 f"Give an example IPv4 address with this regex: {TEST_REGEX}")
68 tensor = torch.rand(32000)
@@ -72,7 +71,6 @@ def test_guided_logits_processors():
71 assert tensor.shape == original_tensor.shape
72 assert not torch.allclose(tensor, original_tensor)
73
75 - json_LP.init_state()
74 token_ids = tokenizer.encode(
75 f"Give an employee profile that fits this schema: {TEST_SCHEMA}")
76 tensor = torch.rand(32000)
vllm/model_executor/guided_decoding/outlines_decoding.py
+12 -19
@@ -1,8 +1,6 @@
1 import asyncio
2 import concurrent.futures
3 -from copy import copy
3 from enum import Enum
5 -from functools import lru_cache
4 from json import dumps as json_dumps
5 from re import escape as regex_escape
6 from typing import Tuple, Union
@@ -54,8 +52,10 @@ global_thread_pool = None # used for generating logits processor fsm
52
53
54 async def get_outlines_guided_decoding_logits_processor(
57 - request: Union[CompletionRequest, ChatCompletionRequest],
58 - tokenizer) -> Union[JSONLogitsProcessor, RegexLogitsProcessor, None]:
55 + request: Union[CompletionRequest,
56 + ChatCompletionRequest], tokenizer: PreTrainedTokenizerBase
57 +) -> Union[JSONLogitsProcessor, RegexLogitsProcessor, CFGLogitsProcessor,
58 + None]:
59 """
60 Given an OpenAI-compatible request, check for guided decoding parameters
61 and get the necessary logits processor for the given guide.
@@ -64,7 +64,7 @@ async def get_outlines_guided_decoding_logits_processor(
64 """
65 global global_thread_pool
66 guide, mode = _get_guide_and_mode(request)
67 - if not guide:
67 + if not guide or not mode:
68 return None
69
70 if global_thread_pool is None:
@@ -72,15 +72,9 @@ async def get_outlines_guided_decoding_logits_processor(
72 max_workers=2)
73 loop = asyncio.get_running_loop()
74
75 - result = await loop.run_in_executor(global_thread_pool,
76 - _get_cached_logits_processor, guide,
77 - tokenizer, mode,
78 - request.guided_whitespace_pattern)
79 -
80 - logits_processor = copy(result)
81 - # reset logits processor's internal state
82 - logits_processor.init_state()
83 - return logits_processor
75 + return await loop.run_in_executor(global_thread_pool,
76 + _get_logits_processor, guide, tokenizer,
77 + mode, request.guided_whitespace_pattern)
78
79
80 def _get_guide_and_mode(
@@ -115,11 +109,10 @@ def _get_guide_and_mode(
109 return None, None
110
111
118 -@lru_cache(maxsize=32)
119 -def _get_cached_logits_processor(guide: str,
120 - tokenizer: PreTrainedTokenizerBase,
121 - mode: GuidedDecodingMode,
122 - whitespace_pattern: Union[str, None]):
112 +def _get_logits_processor(
113 + guide: str, tokenizer: PreTrainedTokenizerBase, mode: GuidedDecodingMode,
114 + whitespace_pattern: Union[str, None]
115 +) -> Union[JSONLogitsProcessor, RegexLogitsProcessor, CFGLogitsProcessor]:
116 if mode == GuidedDecodingMode.JSON:
117 return JSONLogitsProcessor(guide, tokenizer, whitespace_pattern)
118 elif mode == GuidedDecodingMode.REGEX or mode == GuidedDecodingMode.CHOICE:
vllm/model_executor/guided_decoding/outlines_logits_processors.py
+36 -26
@@ -21,7 +21,7 @@ from functools import lru_cache
21 from typing import Callable, DefaultDict, Dict, List, Union
22
23 import torch
24 -from outlines.fsm.fsm import CFGFSM, FSM, RegexFSM
24 +from outlines.fsm.guide import CFGGuide, Generate, Guide, RegexGuide, Write
25 from outlines.fsm.json_schema import build_regex_from_schema
26 from pydantic import BaseModel
27 from transformers import PreTrainedTokenizerBase
@@ -29,28 +29,32 @@ from transformers import PreTrainedTokenizerBase
29
30 class BaseLogitsProcessor:
31
32 - def __init__(self):
33 - # Child class should use initialize in their init.
34 - self.fsm: FSM
35 -
36 - def init_state(self):
37 - """Initialize the FSM states."""
38 - self.fsm_state: DefaultDict[int, int] = defaultdict(int)
32 + def __init__(self, guide: Guide):
33 + self._guide: Guide = guide
34 + self._fsm_state: DefaultDict[int, int] = defaultdict(int)
35
36 def __call__(self, input_ids: List[int],
37 scores: torch.Tensor) -> torch.Tensor:
38 """Use the FSM to bias the logits before sampling the next token."""
39 seq_id = hash(tuple(input_ids))
40
45 - if len(input_ids) == 0:
46 - self.init_state()
47 - else:
41 + if len(input_ids) > 0:
42 last_token = input_ids[-1]
43 last_seq_id = hash(tuple(input_ids[:-1]))
50 - self.fsm_state[seq_id] = self.fsm.next_state(
51 - self.fsm_state[last_seq_id], last_token)
44 + self._fsm_state[seq_id] = self._guide.get_next_state(
45 + state=self._fsm_state[last_seq_id], token_id=last_token)
46 +
47 + instruction = self._guide.get_next_instruction(
48 + state=self._fsm_state[seq_id])
49
53 - allowed_tokens = self.fsm.allowed_token_ids(self.fsm_state[seq_id])
50 + if type(instruction) == Generate:
51 + allowed_tokens = instruction.tokens
52 + elif type(instruction) == Write:
53 + # TODO: support fast forward tokens
54 + allowed_tokens = [instruction.tokens[0]]
55 + else:
56 + raise TypeError(
57 + f"Unsupported instruction type {type(instruction)}")
58
59 mask = torch.full((scores.shape[-1], ),
60 -math.inf,
@@ -62,6 +66,13 @@ class BaseLogitsProcessor:
66
67 class RegexLogitsProcessor(BaseLogitsProcessor):
68
69 + @classmethod
70 + @lru_cache(maxsize=32)
71 + def _get_guide(cls, regex_string: str,
72 + tokenizer: PreTrainedTokenizerBase) -> Guide:
73 + tokenizer = _adapt_tokenizer(tokenizer)
74 + return RegexGuide(regex_string, tokenizer)
75 +
76 def __init__(self, regex_string: str, tokenizer: PreTrainedTokenizerBase):
77 """Compile the FSM that drives the regex-structured generation.
78
@@ -73,9 +84,8 @@ class RegexLogitsProcessor(BaseLogitsProcessor):
84 The model's tokenizer
85
86 """
76 - tokenizer = _adapt_tokenizer(tokenizer)
77 - fsm = RegexFSM(regex_string, tokenizer)
78 - self.fsm = fsm
87 + super().__init__(
88 + RegexLogitsProcessor._get_guide(regex_string, tokenizer))
89
90
91 class JSONLogitsProcessor(RegexLogitsProcessor):
@@ -115,6 +125,12 @@ class JSONLogitsProcessor(RegexLogitsProcessor):
125
126 class CFGLogitsProcessor(BaseLogitsProcessor):
127
128 + @classmethod
129 + @lru_cache(maxsize=32)
130 + def _get_guide(cls, cfg: str, tokenizer: PreTrainedTokenizerBase) -> Guide:
131 + tokenizer = _adapt_tokenizer(tokenizer)
132 + return CFGGuide(cfg, tokenizer)
133 +
134 def __init__(self, cfg: str, tokenizer: PreTrainedTokenizerBase):
135 """Compile the FSM that drives the context free grammar generation.
136
@@ -126,17 +142,11 @@ class CFGLogitsProcessor(BaseLogitsProcessor):
142 The model's tokenizer
143
144 """
129 - tokenizer = _adapt_tokenizer(tokenizer)
130 - fsm = CFGFSM(cfg, tokenizer)
131 - self.fsm = fsm
132 -
133 - def init_state(self):
134 - """Initialize state with a CFGFSM copy."""
135 - super().init_state()
136 - self.fsm = self.fsm.copy()
145 + super().__init__(CFGLogitsProcessor._get_guide(cfg, tokenizer))
146 + self._guide = self._guide.copy()
147
148
139 -@lru_cache
149 +@lru_cache(maxsize=32)
150 def _adapt_tokenizer(tokenizer: PreTrainedTokenizerBase):
151 """Adapt vLLM's tokenizer to use to compile the FSM.
152