@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d81abefd2e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Frontend] add json_schema support from OpenAI protocol (#7654)

Tyler Rockwood committed Aug 24, 2024 at 01:07 UTC d81abefd2ee8e1f4b46b3660ebdaf7b8e19c573a
4 files changed +59 -2
tests/entrypoints/openai/test_chat.py
+33
@@ -837,6 +837,39 @@ async def test_response_format_json_object(client: openai.AsyncOpenAI):
837 assert loaded == {"result": 2}, loaded
838
839
840 +@pytest.mark.asyncio
841 +async def test_response_format_json_schema(client: openai.AsyncOpenAI):
842 + for _ in range(2):
843 + resp = await client.chat.completions.create(
844 + model=MODEL_NAME,
845 + messages=[{
846 + "role":
847 + "user",
848 + "content": ('what is 1+1? please respond with a JSON object, '
849 + 'the format is {"result": 2}')
850 + }],
851 + response_format={
852 + "type": "json_schema",
853 + "json_schema": {
854 + "name": "foo_test",
855 + "schema": {
856 + "type": "object",
857 + "properties": {
858 + "result": {
859 + "type": "integer"
860 + },
861 + },
862 + },
863 + }
864 + })
865 +
866 + content = resp.choices[0].message.content
867 + assert content is not None
868 +
869 + loaded = json.loads(content)
870 + assert loaded == {"result": 2}, loaded
871 +
872 +
873 @pytest.mark.asyncio
874 async def test_extra_fields(client: openai.AsyncOpenAI):
875 with pytest.raises(BadRequestError) as exc_info:
vllm/entrypoints/openai/protocol.py
+12 -2
@@ -85,9 +85,19 @@ class UsageInfo(OpenAIBaseModel):
85 completion_tokens: Optional[int] = 0
86
87
88 +class JsonSchemaResponseFormat(OpenAIBaseModel):
89 + name: str
90 + description: Optional[str] = None
91 + # schema is the field in openai but that causes conflicts with pydantic so
92 + # instead use json_schema with an alias
93 + json_schema: Optional[Dict[str, Any]] = Field(default=None, alias='schema')
94 + strict: Optional[bool] = None
95 +
96 +
97 class ResponseFormat(OpenAIBaseModel):
89 - # type must be "json_object" or "text"
90 - type: Literal["text", "json_object"]
98 + # type must be "json_schema", "json_object" or "text"
99 + type: Literal["text", "json_object", "json_schema"]
100 + json_schema: Optional[JsonSchemaResponseFormat] = None
101
102
103 class StreamOptions(OpenAIBaseModel):
vllm/model_executor/guided_decoding/lm_format_enforcer_decoding.py
+7
@@ -49,6 +49,13 @@ async def get_lm_format_enforcer_guided_decoding_logits_processor(
49 and request.response_format.type == "json_object"):
50 character_level_parser = JsonSchemaParser(
51 None) # None means any json object
52 + elif (request.response_format is not None
53 + and request.response_format.type == "json_schema"
54 + and request.response_format.json_schema is not None
55 + and request.response_format.json_schema.json_schema is not None):
56 + schema = _normalize_json_schema_object(
57 + request.response_format.json_schema.json_schema)
58 + character_level_parser = JsonSchemaParser(schema)
59 else:
60 return None
61
vllm/model_executor/guided_decoding/outlines_decoding.py
+7
@@ -127,6 +127,13 @@ def _get_guide_and_mode(
127 and request.response_format is not None
128 and request.response_format.type == "json_object"):
129 return JSON_GRAMMAR, GuidedDecodingMode.GRAMMAR
130 + elif (not isinstance(request, GuidedDecodingRequest)
131 + and request.response_format is not None
132 + and request.response_format.type == "json_schema"
133 + and request.response_format.json_schema is not None
134 + and request.response_format.json_schema.json_schema is not None):
135 + json = json_dumps(request.response_format.json_schema.json_schema)
136 + return json, GuidedDecodingMode.JSON
137 else:
138 return None, None
139