@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5b59fe0f08
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Bugfix] Pass json-schema to GuidedDecodingParams and make test stronger (#9530)

Chen Zhang committed Oct 19, 2024 at 17:05 UTC 5b59fe0f08c16e56813f2dad442d44cab222668b
2 files changed +29 -9
tests/entrypoints/openai/test_chat.py
+18 -4
@@ -851,14 +851,28 @@ async def test_response_format_json_object(client: openai.AsyncOpenAI):
851
852 @pytest.mark.asyncio
853 async def test_response_format_json_schema(client: openai.AsyncOpenAI):
854 + prompt = 'what is 1+1? The format is "result": 2'
855 + # Check that this prompt cannot lead to a valid JSON without json_schema
856 for _ in range(2):
857 resp = await client.chat.completions.create(
858 model=MODEL_NAME,
859 messages=[{
858 - "role":
859 - "user",
860 - "content": ('what is 1+1? please respond with a JSON object, '
861 - 'the format is {"result": 2}')
860 + "role": "user",
861 + "content": prompt
862 + }],
863 + )
864 + content = resp.choices[0].message.content
865 + assert content is not None
866 + with pytest.raises((json.JSONDecodeError, AssertionError)):
867 + loaded = json.loads(content)
868 + assert loaded == {"result": 2}, loaded
869 +
870 + for _ in range(2):
871 + resp = await client.chat.completions.create(
872 + model=MODEL_NAME,
873 + messages=[{
874 + "role": "user",
875 + "content": prompt
876 }],
877 response_format={
878 "type": "json_schema",
vllm/entrypoints/openai/protocol.py
+11 -5
@@ -314,9 +314,15 @@ class ChatCompletionRequest(OpenAIBaseModel):
314 prompt_logprobs = self.top_logprobs
315
316 guided_json_object = None
317 - if (self.response_format is not None
318 - and self.response_format.type == "json_object"):
319 - guided_json_object = True
317 + if self.response_format is not None:
318 + if self.response_format.type == "json_object":
319 + guided_json_object = True
320 + elif self.response_format.type == "json_schema":
321 + json_schema = self.response_format.json_schema
322 + assert json_schema is not None
323 + self.guided_json = json_schema.json_schema
324 + if self.guided_decoding_backend is None:
325 + self.guided_decoding_backend = "lm-format-enforcer"
326
327 guided_decoding = GuidedDecodingParams.from_optional(
328 json=self._get_guided_json_from_tool() or self.guided_json,
@@ -537,8 +543,8 @@ class CompletionRequest(OpenAIBaseModel):
543 default=None,
544 description=
545 ("Similar to chat completion, this parameter specifies the format of "
540 - "output. Only {'type': 'json_object'} or {'type': 'text' } is "
541 - "supported."),
546 + "output. Only {'type': 'json_object'}, {'type': 'json_schema'} or "
547 + "{'type': 'text' } is supported."),
548 )
549 guided_json: Optional[Union[str, dict, BaseModel]] = Field(
550 default=None,