@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 81fbb3655f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[CI/Build] Test both text and token IDs in batched OpenAI Completions API (#5568)

Cyrus Leung committed Jun 15, 2024 at 19:29 UTC 81fbb3655f37e2b3ccbe0e17276c5d813b886417
1 file changed +45 -43
tests/entrypoints/test_openai_server.py
+45 -43
@@ -655,50 +655,52 @@ async def test_completion_stream_options(client: openai.AsyncOpenAI,
655 [MODEL_NAME, "zephyr-lora"],
656 )
657 async def test_batch_completions(client: openai.AsyncOpenAI, model_name: str):
658 - # test simple list
659 - batch = await client.completions.create(
660 - model=model_name,
661 - prompt=["Hello, my name is", "Hello, my name is"],
662 - max_tokens=5,
663 - temperature=0.0,
664 - )
665 - assert len(batch.choices) == 2
666 - assert batch.choices[0].text == batch.choices[1].text
667 -
668 - # test n = 2
669 - batch = await client.completions.create(
670 - model=model_name,
671 - prompt=["Hello, my name is", "Hello, my name is"],
672 - n=2,
673 - max_tokens=5,
674 - temperature=0.0,
675 - extra_body=dict(
676 - # NOTE: this has to be true for n > 1 in vLLM, but not necessary
677 - # for official client.
678 - use_beam_search=True),
679 - )
680 - assert len(batch.choices) == 4
681 - assert batch.choices[0].text != batch.choices[
682 - 1].text, "beam search should be different"
683 - assert batch.choices[0].text == batch.choices[
684 - 2].text, "two copies of the same prompt should be the same"
685 - assert batch.choices[1].text == batch.choices[
686 - 3].text, "two copies of the same prompt should be the same"
658 + # test both text and token IDs
659 + for prompts in (["Hello, my name is"] * 2, [[0, 0, 0, 0, 0]] * 2):
660 + # test simple list
661 + batch = await client.completions.create(
662 + model=model_name,
663 + prompt=prompts,
664 + max_tokens=5,
665 + temperature=0.0,
666 + )
667 + assert len(batch.choices) == 2
668 + assert batch.choices[0].text == batch.choices[1].text
669
688 - # test streaming
689 - batch = await client.completions.create(
690 - model=model_name,
691 - prompt=["Hello, my name is", "Hello, my name is"],
692 - max_tokens=5,
693 - temperature=0.0,
694 - stream=True,
695 - )
696 - texts = [""] * 2
697 - async for chunk in batch:
698 - assert len(chunk.choices) == 1
699 - choice = chunk.choices[0]
700 - texts[choice.index] += choice.text
701 - assert texts[0] == texts[1]
670 + # test n = 2
671 + batch = await client.completions.create(
672 + model=model_name,
673 + prompt=prompts,
674 + n=2,
675 + max_tokens=5,
676 + temperature=0.0,
677 + extra_body=dict(
678 + # NOTE: this has to be true for n > 1 in vLLM, but not necessary
679 + # for official client.
680 + use_beam_search=True),
681 + )
682 + assert len(batch.choices) == 4
683 + assert batch.choices[0].text != batch.choices[
684 + 1].text, "beam search should be different"
685 + assert batch.choices[0].text == batch.choices[
686 + 2].text, "two copies of the same prompt should be the same"
687 + assert batch.choices[1].text == batch.choices[
688 + 3].text, "two copies of the same prompt should be the same"
689 +
690 + # test streaming
691 + batch = await client.completions.create(
692 + model=model_name,
693 + prompt=prompts,
694 + max_tokens=5,
695 + temperature=0.0,
696 + stream=True,
697 + )
698 + texts = [""] * 2
699 + async for chunk in batch:
700 + assert len(chunk.choices) == 1
701 + choice = chunk.choices[0]
702 + texts[choice.index] += choice.text
703 + assert texts[0] == texts[1]
704
705
706 @pytest.mark.asyncio