A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Frontend] add json_schema support from OpenAI protocol (#7654)
Tyler Rockwood committed
Aug 24, 2024 at 01:07 UTC
d81abefd2ee8e1f4b46b3660ebdaf7b8e19c573a
4 files changed
+59
-2
tests/entrypoints/openai/test_chat.py
+33
@@ -837,6 +837,39 @@ async def test_response_format_json_object(client: openai.AsyncOpenAI):
837
assert loaded == {"result": 2}, loaded
838
839
840
+@pytest.mark.asyncio
841
+async def test_response_format_json_schema(client: openai.AsyncOpenAI):
842
+ for _ in range(2):
843
+ resp = await client.chat.completions.create(
844
+ model=MODEL_NAME,
845
+ messages=[{
846
+ "role":
847
+ "user",
848
+ "content": ('what is 1+1? please respond with a JSON object, '
849
+ 'the format is {"result": 2}')
850
+ }],
851
+ response_format={
852
+ "type": "json_schema",
853
+ "json_schema": {
854
+ "name": "foo_test",
855
+ "schema": {
856
+ "type": "object",
857
+ "properties": {
858
+ "result": {
859
+ "type": "integer"
860
+ },
861
+ },
862
+ },
863
+ }
864
+ })
865
+
866
+ content = resp.choices[0].message.content
867
+ assert content is not None
868
+
869
+ loaded = json.loads(content)
870
+ assert loaded == {"result": 2}, loaded
871
+
872
+
873
@pytest.mark.asyncio
874
async def test_extra_fields(client: openai.AsyncOpenAI):
875
with pytest.raises(BadRequestError) as exc_info:
vllm/entrypoints/openai/protocol.py
+12
-2
@@ -85,9 +85,19 @@ class UsageInfo(OpenAIBaseModel):
85
completion_tokens: Optional[int] = 0
86
87
88
+class JsonSchemaResponseFormat(OpenAIBaseModel):
89
+ name: str
90
+ description: Optional[str] = None
91
+ # schema is the field in openai but that causes conflicts with pydantic so
92
+ # instead use json_schema with an alias
93
+ json_schema: Optional[Dict[str, Any]] = Field(default=None, alias='schema')
94
+ strict: Optional[bool] = None
95
+
96
+
97
class ResponseFormat(OpenAIBaseModel):
89
- # type must be "json_object" or "text"
90
- type: Literal["text", "json_object"]
98
+ # type must be "json_schema", "json_object" or "text"
99
+ type: Literal["text", "json_object", "json_schema"]
100
+ json_schema: Optional[JsonSchemaResponseFormat] = None
101
102
103
class StreamOptions(OpenAIBaseModel):
vllm/model_executor/guided_decoding/lm_format_enforcer_decoding.py
+7
@@ -49,6 +49,13 @@ async def get_lm_format_enforcer_guided_decoding_logits_processor(
49
and request.response_format.type == "json_object"):
50
character_level_parser = JsonSchemaParser(
51
None) # None means any json object
52
+ elif (request.response_format is not None
53
+ and request.response_format.type == "json_schema"
54
+ and request.response_format.json_schema is not None
55
+ and request.response_format.json_schema.json_schema is not None):
56
+ schema = _normalize_json_schema_object(
57
+ request.response_format.json_schema.json_schema)
58
+ character_level_parser = JsonSchemaParser(schema)
59
else:
60
return None
61
vllm/model_executor/guided_decoding/outlines_decoding.py
+7
@@ -127,6 +127,13 @@ def _get_guide_and_mode(
127
and request.response_format is not None
128
and request.response_format.type == "json_object"):
129
return JSON_GRAMMAR, GuidedDecodingMode.GRAMMAR
130
+ elif (not isinstance(request, GuidedDecodingRequest)
131
+ and request.response_format is not None
132
+ and request.response_format.type == "json_schema"
133
+ and request.response_format.json_schema is not None
134
+ and request.response_format.json_schema.json_schema is not None):
135
+ json = json_dumps(request.response_format.json_schema.json_schema)
136
+ return json, GuidedDecodingMode.JSON
137
else:
138
return None, None
139