A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Support arbitrary json_object in OpenAI and Context Free Grammar (#3211)
Simon Mo committed
Mar 16, 2024 at 13:35 UTC
120157fd2a256faf9e4d9941aa580c195735b878
4 files changed
+176
-49
tests/entrypoints/test_openai_server.py
+50
@@ -660,5 +660,55 @@ async def test_guided_decoding_type_error(server, client: openai.AsyncOpenAI):
660
extra_body=dict(guided_regex=TEST_REGEX, guided_json=TEST_SCHEMA))
661
662
663
+async def test_response_format_json_object(server, client: openai.AsyncOpenAI):
664
+ resp = await client.chat.completions.create(
665
+ model=MODEL_NAME,
666
+ messages=[{
667
+ "role":
668
+ "user",
669
+ "content": ('what is 1+1? please respond with a JSON object, '
670
+ 'the format is {"result": 2}')
671
+ }],
672
+ response_format={"type": "json_object"})
673
+
674
+ content = resp.choices[0].message.content
675
+ loaded = json.loads(content)
676
+ assert loaded == {"result": 2}, loaded
677
+
678
+
679
+async def test_guided_grammar(server, client: openai.AsyncOpenAI):
680
+ simple_sql_grammar = """
681
+start: select_statement
682
+
683
+select_statement: "SELECT" column "from" table "where" condition
684
+
685
+column: "col_1" | "col_2"
686
+table: "table_1" | "table_2"
687
+condition: column "=" number
688
+
689
+number: "1" | "2"
690
+"""
691
+
692
+ completion = await client.completions.create(
693
+ model=MODEL_NAME,
694
+ prompt=("Generate a sql state that select col_1 from "
695
+ "table_1 where it is equals to 1"),
696
+ temperature=1.0,
697
+ max_tokens=500,
698
+ extra_body=dict(guided_grammar=simple_sql_grammar))
699
+
700
+ content = completion.choices[0].text
701
+
702
+ # use Lark to parse the output, and make sure it's a valid parse tree
703
+ from lark import Lark
704
+ parser = Lark(simple_sql_grammar)
705
+ parser.parse(content)
706
+
707
+ # remove spaces for comparison b/c we removed them in the grammar
708
+ ground_truth = "SELECT col_1 from table_1 where col_1 = 1".replace(" ", "")
709
+
710
+ assert content.strip() == ground_truth
711
+
712
+
713
if __name__ == "__main__":
714
pytest.main([__file__])
vllm/entrypoints/openai/protocol.py
+9
@@ -55,6 +55,11 @@ class UsageInfo(BaseModel):
55
completion_tokens: Optional[int] = 0
56
57
58
+class ResponseFormat(BaseModel):
59
+ # type must be "json_object" or "text"
60
+ type: str = Literal["text", "json_object"]
61
+
62
+
63
class ChatCompletionRequest(BaseModel):
64
model: str
65
messages: List[Dict[str, str]]
@@ -89,6 +94,8 @@ class ChatCompletionRequest(BaseModel):
94
guided_json: Optional[Union[str, dict, BaseModel]] = None
95
guided_regex: Optional[str] = None
96
guided_choice: Optional[List[str]] = None
97
+ guided_grammar: Optional[str] = None
98
+ response_format: Optional[ResponseFormat] = None
99
100
def to_sampling_params(self) -> SamplingParams:
101
if self.logprobs and not self.top_logprobs:
@@ -183,6 +190,8 @@ class CompletionRequest(BaseModel):
190
guided_json: Optional[Union[str, dict, BaseModel]] = None
191
guided_regex: Optional[str] = None
192
guided_choice: Optional[List[str]] = None
193
+ guided_grammar: Optional[str] = None
194
+ response_format: Optional[ResponseFormat] = None
195
196
def to_sampling_params(self):
197
echo_without_generation = self.echo and self.max_tokens == 0
vllm/model_executor/guided_decoding.py
+41
-13
@@ -6,19 +6,50 @@ from functools import lru_cache
6
from json import dumps as json_dumps
7
from re import escape as regex_escape
8
from typing import Union, Tuple
9
+
10
from pydantic import BaseModel
11
+from transformers import PreTrainedTokenizerBase
12
13
from vllm.entrypoints.openai.protocol import (CompletionRequest,
14
ChatCompletionRequest)
15
from vllm.model_executor.guided_logits_processors import (JSONLogitsProcessor,
14
- RegexLogitsProcessor)
16
+ RegexLogitsProcessor,
17
+ CFGLogitsProcessor)
18
19
20
class GuidedDecodingMode(Enum):
21
JSON = "json"
22
REGEX = "regex"
23
CHOICE = "choice"
24
+ GRAMMAR = "grammar"
25
+
26
+
27
+# https://github.com/outlines-dev/outlines/blob/main/outlines/grammars/json.lark
28
+# the main difference is that we changed the start: value to
29
+# start: object | array, so we are denying scalar values as the root of the
30
+# JSON. Starting with scalars as the root seems to cause llama to generate
31
+# without stop.
32
+JSON_GRAMMAR = r"""
33
+?start: object | array
34
+
35
+?value: object
36
+| array
37
+| UNESCAPED_STRING
38
+| SIGNED_NUMBER -> number
39
+| "true" -> true
40
+| "false" -> false
41
+| "null" -> null
42
+
43
+array : "[" [value ("," value)*] "]"
44
+object : "{" [pair ("," pair)*] "}"
45
+pair : UNESCAPED_STRING ":" value
46
+
47
+%import common.UNESCAPED_STRING
48
+%import common.SIGNED_NUMBER
49
+%import common.WS
50
51
+%ignore WS
52
+"""
53
54
global_thread_pool = None # used for generating logits processor fsm
55
@@ -57,9 +88,6 @@ def _get_guide_and_mode(
88
) -> Tuple[str, GuidedDecodingMode]:
89
90
if request.guided_json:
60
- if not isinstance(request.guided_json, (str, dict, BaseModel)):
61
- raise TypeError("JSON schema must be str, dict, or BaseModel")
62
-
91
json = request.guided_json
92
if isinstance(json, dict):
93
# turn dict into hashable string
@@ -69,33 +97,33 @@ def _get_guide_and_mode(
97
# with the same fields will get hashed the same
98
json = str(json.__signature__)
99
return json, GuidedDecodingMode.JSON
72
-
100
elif request.guided_regex:
74
- if not isinstance(request.guided_regex, str):
75
- raise TypeError("Regex must be string")
101
return request.guided_regex, GuidedDecodingMode.REGEX
77
-
102
elif request.guided_choice:
79
- if not isinstance(request.guided_choice, list):
80
- raise TypeError("Choices must be a list")
81
-
103
# choice just uses regex
104
choices = [
105
regex_escape(str(choice)) for choice in request.guided_choice
106
]
107
choices_regex = "(" + "|".join(choices) + ")"
108
return choices_regex, GuidedDecodingMode.CHOICE
88
-
109
+ elif request.guided_grammar:
110
+ return request.guided_grammar, GuidedDecodingMode.GRAMMAR
111
+ elif (request.response_format is not None
112
+ and request.response_format.type == "json_object"):
113
+ return JSON_GRAMMAR, GuidedDecodingMode.GRAMMAR
114
else:
115
return None, None
116
117
118
@lru_cache(maxsize=32)
94
-def _get_cached_logits_processor(guide: str, tokenizer,
119
+def _get_cached_logits_processor(guide: str,
120
+ tokenizer: PreTrainedTokenizerBase,
121
mode: GuidedDecodingMode):
122
if mode == GuidedDecodingMode.JSON:
123
return JSONLogitsProcessor(guide, tokenizer)
124
elif mode == GuidedDecodingMode.REGEX or mode == GuidedDecodingMode.CHOICE:
125
return RegexLogitsProcessor(guide, tokenizer)
126
+ elif mode == GuidedDecodingMode.GRAMMAR:
127
+ return CFGLogitsProcessor(guide, tokenizer)
128
else:
129
raise ValueError(f"Unknown guided decoding mode {mode}")
vllm/model_executor/guided_logits_processors.py
+76
-36
@@ -16,30 +16,60 @@
16
import json
17
import math
18
from collections import defaultdict
19
-from typing import Union, DefaultDict, Dict, List, Optional
19
+from typing import Union, DefaultDict, Dict, List, Optional, Callable
20
21
import torch
22
from pydantic import BaseModel
23
-from outlines.fsm.fsm import RegexFSM
23
+from transformers import PreTrainedTokenizerBase
24
+from outlines.fsm.fsm import RegexFSM, CFGFSM
25
from outlines.fsm.json_schema import build_regex_from_schema
26
27
27
-class RegexLogitsProcessor:
28
+class BaseLogitsProcessor:
29
29
- def __init__(self, regex_string: str, tokenizer):
30
- """Compile the FSM that drives the regex-structured generation.
30
+ def adapt_tokenizer(self, tokenizer: PreTrainedTokenizerBase):
31
+ """Adapt vLLM's tokenizer to use to compile the FSM.
32
32
- Parameters
33
- ----------
34
- regex_string
35
- A string that represents a regular expression
36
- tokenizer
37
- The model's tokenizer
33
+ The API of Outlines tokenizers is slightly different to that of
34
+ `transformers`. The decoder of outlines, returns a list whereas
35
+ the decode of vLLM returns an str. To sync the vLLM decoder with
36
+ outlines internal api, the decoder should be adapted. In addition
37
+ we need to handle the missing spaces to Llama's tokenizer to be
38
+ able to compile FSMs for this model.
39
40
"""
40
- tokenizer = self.adapt_tokenizer(tokenizer)
41
- fsm = RegexFSM(regex_string, tokenizer)
42
- self.fsm = fsm
41
+ if getattr(tokenizer, "_outlines_adapted", False):
42
+ return tokenizer
43
+
44
+ tokenizer.vocabulary = tokenizer.get_vocab()
45
+ tokenizer.special_tokens = set(tokenizer.all_special_tokens)
46
+
47
+ def convert_token_to_string(token: str) -> str:
48
+ from transformers.file_utils import SPIECE_UNDERLINE
49
+
50
+ string = tokenizer.convert_tokens_to_string([token])
51
+
52
+ # A hack to handle missing spaces to HF's Llama tokenizers
53
+ if token.startswith(SPIECE_UNDERLINE) or token == "<0x20>":
54
+ return " " + string
55
+
56
+ return string
57
+
58
+ def change_decoder(
59
+ decoder: Callable[[List[int]], str]
60
+ ) -> Callable[[List[int]], List[str]]:
61
+ """Sync vLLM's decoder with the outlines by returning list."""
62
+
63
+ def new_decoder(inp_tokens: List[int]) -> List[str]:
64
+ return [decoder(inp_tokens)]
65
+
66
+ return new_decoder
67
+
68
+ tokenizer.convert_token_to_string = convert_token_to_string
69
+ tokenizer.decode = change_decoder(tokenizer.decode)
70
+ setattr(tokenizer, "_outlines_adapted", True) # noqa: B010
71
+
72
+ return tokenizer
73
74
def init_state(self):
75
"""Initialize the FSM states."""
@@ -69,38 +99,30 @@ class RegexLogitsProcessor:
99
100
return scores
101
72
- def adapt_tokenizer(self, tokenizer):
73
- """Adapt vLLM's tokenizer to use to compile the FSM.
74
-
75
- The API of Outlines tokenizers is slightly different to that of
76
- `transformers`. In addition we need to handle the missing spaces to
77
- Llama's tokenizer to be able to compile FSMs for this model.
78
-
79
- """
80
- tokenizer.vocabulary = tokenizer.get_vocab()
81
- tokenizer.special_tokens = set(tokenizer.all_special_tokens)
82
-
83
- def convert_token_to_string(token: str) -> str:
84
- from transformers.file_utils import SPIECE_UNDERLINE
102
86
- string = tokenizer.convert_tokens_to_string([token])
103
+class RegexLogitsProcessor(BaseLogitsProcessor):
104
88
- # A hack to handle missing spaces to HF's Llama tokenizers
89
- if token.startswith(SPIECE_UNDERLINE) or token == "<0x20>":
90
- return " " + string
91
-
92
- return string
105
+ def __init__(self, regex_string: str, tokenizer: PreTrainedTokenizerBase):
106
+ """Compile the FSM that drives the regex-structured generation.
107
94
- tokenizer.convert_token_to_string = convert_token_to_string
108
+ Parameters
109
+ ----------
110
+ regex_string
111
+ A string that represents a regular expression
112
+ tokenizer
113
+ The model's tokenizer
114
96
- return tokenizer
115
+ """
116
+ tokenizer = self.adapt_tokenizer(tokenizer)
117
+ fsm = RegexFSM(regex_string, tokenizer)
118
+ self.fsm = fsm
119
120
121
class JSONLogitsProcessor(RegexLogitsProcessor):
122
123
def __init__(self,
124
schema: Union[str, Dict, BaseModel],
103
- tokenizer,
125
+ tokenizer: PreTrainedTokenizerBase,
126
whitespace_pattern: Optional[str] = None):
127
"""Compile the FSM that drives the JSON-guided generation.
128
@@ -130,3 +152,21 @@ class JSONLogitsProcessor(RegexLogitsProcessor):
152
f"the JSON Schema specification")
153
regex_string = build_regex_from_schema(schema_str, whitespace_pattern)
154
super().__init__(regex_string, tokenizer)
155
+
156
+
157
+class CFGLogitsProcessor(BaseLogitsProcessor):
158
+
159
+ def __init__(self, cfg: str, tokenizer: PreTrainedTokenizerBase):
160
+ """Compile the FSM that drives the context free grammar generation.
161
+
162
+ Parameters
163
+ ----------
164
+ cfg
165
+ A string that represents a context-free grammar
166
+ tokenizer
167
+ The model's tokenizer
168
+
169
+ """
170
+ tokenizer = self.adapt_tokenizer(tokenizer)
171
+ fsm = CFGFSM(cfg, tokenizer)
172
+ self.fsm = fsm