@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e1bb2fd52d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Bugfix] Support logprobs when using guided_json and other constrained decoding fields (#4149)

James Whedbee committed Apr 18, 2024 at 16:12 UTC e1bb2fd52dea0bbc772bdf35fd27664c5daec7b2
2 files changed +33 -1
tests/entrypoints/test_openai_server.py
+30
@@ -723,6 +723,36 @@ async def test_guided_decoding_type_error(server, client: openai.AsyncOpenAI,
723 extra_body=dict(guided_regex=TEST_REGEX, guided_json=TEST_SCHEMA))
724
725
726 +@pytest.mark.parametrize("guided_decoding_backend",
727 + ["outlines", "lm-format-enforcer"])
728 +async def test_guided_choice_chat_logprobs(server, client: openai.AsyncOpenAI,
729 + guided_decoding_backend: str):
730 + messages = [{
731 + "role": "system",
732 + "content": "you are a helpful assistant"
733 + }, {
734 + "role":
735 + "user",
736 + "content":
737 + "The best language for type-safe systems programming is "
738 + }]
739 + chat_completion = await client.chat.completions.create(
740 + model=MODEL_NAME,
741 + messages=messages,
742 + max_tokens=10,
743 + logprobs=True,
744 + top_logprobs=5,
745 + extra_body=dict(guided_choice=TEST_CHOICE,
746 + guided_decoding_backend=guided_decoding_backend))
747 + top_logprobs = chat_completion.choices[0].logprobs.top_logprobs
748 +
749 + # -9999.0 is the minimum logprob returned by OpenAI
750 + assert all(
751 + isinstance(logprob, float) and logprob >= -9999.0
752 + for token_dict in top_logprobs
753 + for token, logprob in token_dict.items())
754 +
755 +
756 async def test_response_format_json_object(server, client: openai.AsyncOpenAI):
757 resp = await client.chat.completions.create(
758 model=MODEL_NAME,
vllm/entrypoints/openai/serving_engine.py
+3 -1
@@ -116,7 +116,9 @@ class OpenAIServing:
116
117 if num_output_top_logprobs:
118 logprobs.top_logprobs.append({
119 - p.decoded_token: p.logprob
119 + # Convert float("-inf") to the
120 + # JSON-serializable float that OpenAI uses
121 + p.decoded_token: max(p.logprob, -9999.0)
122 for i, p in step_top_logprobs.items()
123 } if step_top_logprobs else None)
124