A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[CI] Expand OpenAI test_chat.py guided decoding tests (#11048)
Signed-off-by: mgoin <michael@neuralmagic.com>
Michael Goin committed
Dec 23, 2024 at 13:35 UTC
63afbe9215813780e0327e31072c4292bd99e46b
1 file changed
+12
-17
tests/entrypoints/openai/test_chat.py
+12
-17
@@ -17,6 +17,8 @@ from .test_completion import zephyr_lora_files # noqa: F401
17
# any model with a chat template should work here
18
MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
19
20
+GUIDED_DECODING_BACKENDS = ["outlines", "lm-format-enforcer", "xgrammar"]
21
+
22
23
@pytest.fixture(scope="module")
24
def server(zephyr_lora_files, zephyr_lora_added_tokens_files): # noqa: F811
@@ -464,8 +466,7 @@ async def test_chat_completion_stream_options(client: openai.AsyncOpenAI,
466
# will fail on the second `guided_decoding_backend` even when I swap their order
467
# (ref: https://github.com/vllm-project/vllm/pull/5526#issuecomment-2173772256)
468
@pytest.mark.asyncio
467
-@pytest.mark.parametrize("guided_decoding_backend",
468
- ["outlines", "lm-format-enforcer"])
469
+@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)
470
async def test_guided_choice_chat(client: openai.AsyncOpenAI,
471
guided_decoding_backend: str,
472
sample_guided_choice):
@@ -506,8 +507,7 @@ async def test_guided_choice_chat(client: openai.AsyncOpenAI,
507
508
509
@pytest.mark.asyncio
509
-@pytest.mark.parametrize("guided_decoding_backend",
510
- ["outlines", "lm-format-enforcer"])
510
+@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)
511
async def test_guided_json_chat(client: openai.AsyncOpenAI,
512
guided_decoding_backend: str,
513
sample_json_schema):
@@ -554,8 +554,7 @@ async def test_guided_json_chat(client: openai.AsyncOpenAI,
554
555
556
@pytest.mark.asyncio
557
-@pytest.mark.parametrize("guided_decoding_backend",
558
- ["outlines", "lm-format-enforcer"])
557
+@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)
558
async def test_guided_regex_chat(client: openai.AsyncOpenAI,
559
guided_decoding_backend: str, sample_regex):
560
messages = [{
@@ -613,8 +612,7 @@ async def test_guided_decoding_type_error(client: openai.AsyncOpenAI):
612
613
614
@pytest.mark.asyncio
616
-@pytest.mark.parametrize("guided_decoding_backend",
617
- ["outlines", "lm-format-enforcer"])
615
+@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)
616
async def test_guided_choice_chat_logprobs(client: openai.AsyncOpenAI,
617
guided_decoding_backend: str,
618
sample_guided_choice):
@@ -646,8 +644,7 @@ async def test_guided_choice_chat_logprobs(client: openai.AsyncOpenAI,
644
645
646
@pytest.mark.asyncio
649
-@pytest.mark.parametrize("guided_decoding_backend",
650
- ["outlines", "lm-format-enforcer"])
647
+@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)
648
async def test_named_tool_use(client: openai.AsyncOpenAI,
649
guided_decoding_backend: str,
650
sample_json_schema):
@@ -681,7 +678,8 @@ async def test_named_tool_use(client: openai.AsyncOpenAI,
678
"function": {
679
"name": "dummy_function_name"
680
}
684
- })
681
+ },
682
+ extra_body=dict(guided_decoding_backend=guided_decoding_backend))
683
message = chat_completion.choices[0].message
684
assert len(message.content) == 0
685
json_string = message.tool_calls[0].function.arguments
@@ -716,6 +714,7 @@ async def test_named_tool_use(client: openai.AsyncOpenAI,
714
"name": "dummy_function_name"
715
}
716
},
717
+ extra_body=dict(guided_decoding_backend=guided_decoding_backend),
718
stream=True)
719
720
output = []
@@ -738,10 +737,8 @@ async def test_named_tool_use(client: openai.AsyncOpenAI,
737
738
739
@pytest.mark.asyncio
741
-@pytest.mark.parametrize("guided_decoding_backend", ["outlines"])
742
-async def test_required_tool_use_not_yet_supported(
743
- client: openai.AsyncOpenAI, guided_decoding_backend: str,
744
- sample_json_schema):
740
+async def test_required_tool_use_not_yet_supported(client: openai.AsyncOpenAI,
741
+ sample_json_schema):
742
messages = [{
743
"role": "system",
744
"content": "you are a helpful assistant"
@@ -785,9 +782,7 @@ async def test_required_tool_use_not_yet_supported(
782
783
784
@pytest.mark.asyncio
788
-@pytest.mark.parametrize("guided_decoding_backend", ["outlines"])
785
async def test_inconsistent_tool_choice_and_tools(client: openai.AsyncOpenAI,
790
- guided_decoding_backend: str,
786
sample_json_schema):
787
messages = [{
788
"role": "system",