A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Bugfix] Pass json-schema to GuidedDecodingParams and make test stronger (#9530)
Chen Zhang committed
Oct 19, 2024 at 17:05 UTC
5b59fe0f08c16e56813f2dad442d44cab222668b
2 files changed
+29
-9
tests/entrypoints/openai/test_chat.py
+18
-4
@@ -851,14 +851,28 @@ async def test_response_format_json_object(client: openai.AsyncOpenAI):
851
852
@pytest.mark.asyncio
853
async def test_response_format_json_schema(client: openai.AsyncOpenAI):
854
+ prompt = 'what is 1+1? The format is "result": 2'
855
+ # Check that this prompt cannot lead to a valid JSON without json_schema
856
for _ in range(2):
857
resp = await client.chat.completions.create(
858
model=MODEL_NAME,
859
messages=[{
858
- "role":
859
- "user",
860
- "content": ('what is 1+1? please respond with a JSON object, '
861
- 'the format is {"result": 2}')
860
+ "role": "user",
861
+ "content": prompt
862
+ }],
863
+ )
864
+ content = resp.choices[0].message.content
865
+ assert content is not None
866
+ with pytest.raises((json.JSONDecodeError, AssertionError)):
867
+ loaded = json.loads(content)
868
+ assert loaded == {"result": 2}, loaded
869
+
870
+ for _ in range(2):
871
+ resp = await client.chat.completions.create(
872
+ model=MODEL_NAME,
873
+ messages=[{
874
+ "role": "user",
875
+ "content": prompt
876
}],
877
response_format={
878
"type": "json_schema",
vllm/entrypoints/openai/protocol.py
+11
-5
@@ -314,9 +314,15 @@ class ChatCompletionRequest(OpenAIBaseModel):
314
prompt_logprobs = self.top_logprobs
315
316
guided_json_object = None
317
- if (self.response_format is not None
318
- and self.response_format.type == "json_object"):
319
- guided_json_object = True
317
+ if self.response_format is not None:
318
+ if self.response_format.type == "json_object":
319
+ guided_json_object = True
320
+ elif self.response_format.type == "json_schema":
321
+ json_schema = self.response_format.json_schema
322
+ assert json_schema is not None
323
+ self.guided_json = json_schema.json_schema
324
+ if self.guided_decoding_backend is None:
325
+ self.guided_decoding_backend = "lm-format-enforcer"
326
327
guided_decoding = GuidedDecodingParams.from_optional(
328
json=self._get_guided_json_from_tool() or self.guided_json,
@@ -537,8 +543,8 @@ class CompletionRequest(OpenAIBaseModel):
543
default=None,
544
description=
545
("Similar to chat completion, this parameter specifies the format of "
540
- "output. Only {'type': 'json_object'} or {'type': 'text' } is "
541
- "supported."),
546
+ "output. Only {'type': 'json_object'}, {'type': 'json_schema'} or "
547
+ "{'type': 'text' } is supported."),
548
)
549
guided_json: Optional[Union[str, dict, BaseModel]] = Field(
550
default=None,