@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / da7e702c6f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[Bug]: When apply continue_final_message for OpenAI server, the "echo":false is ignored (#10180)

Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>

Chauncey committed Nov 22, 2024 at 00:24 UTC da7e702c6fae521bf8633affb8fe7b834f5cb94b
2 files changed +81 -2
tests/entrypoints/openai/test_chat_echo.py new
+79
@@ -0,0 +1,79 @@
1 +from typing import NamedTuple
2 +
3 +import openai # use the official client for correctness check
4 +import pytest
5 +import pytest_asyncio
6 +
7 +from ...utils import RemoteOpenAIServer
8 +
9 +# # any model with a chat template should work here
10 +MODEL_NAME = "Qwen/Qwen2-1.5B-Instruct"
11 +DUMMY_CHAT_TEMPLATE = """{% for message in messages %}{{message['role'] + ': ' + message['content'] + '\\n'}}{% endfor %}""" # noqa: E501
12 +
13 +
14 +@pytest.fixture(scope="module")
15 +def server():
16 + args = [
17 + # use half precision for speed and memory savings in CI environment
18 + "--dtype",
19 + "float16",
20 + "--enforce-eager",
21 + "--max-model-len",
22 + "4080",
23 + "--chat-template",
24 + DUMMY_CHAT_TEMPLATE,
25 + ]
26 +
27 + with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
28 + yield remote_server
29 +
30 +
31 +@pytest_asyncio.fixture
32 +async def client(server):
33 + async with server.get_async_client() as async_client:
34 + yield async_client
35 +
36 +
37 +class TestCase(NamedTuple):
38 + model_name: str
39 + echo: bool
40 +
41 +
42 +@pytest.mark.asyncio
43 +@pytest.mark.parametrize(
44 + "test_case",
45 + [
46 + TestCase(model_name=MODEL_NAME, echo=True),
47 + TestCase(model_name=MODEL_NAME, echo=False)
48 + ],
49 +)
50 +async def test_chat_session_with_echo_and_continue_final_message(
51 + client: openai.AsyncOpenAI, test_case: TestCase):
52 + saying: str = "Here is a common saying about apple. An apple a day, keeps"
53 + # test echo with continue_final_message parameter
54 + chat_completion = await client.chat.completions.create(
55 + model=test_case.model_name,
56 + messages=[{
57 + "role": "user",
58 + "content": "tell me a common saying"
59 + }, {
60 + "role": "assistant",
61 + "content": saying
62 + }],
63 + extra_body={
64 + "echo": test_case.echo,
65 + "continue_final_message": True,
66 + "add_generation_prompt": False
67 + })
68 + assert chat_completion.id is not None
69 + assert len(chat_completion.choices) == 1
70 +
71 + choice = chat_completion.choices[0]
72 + assert choice.finish_reason == "stop"
73 +
74 + message = choice.message
75 + if test_case.echo:
76 + assert message.content is not None and saying in message.content
77 + else:
78 + assert message.content is not None and saying not in message.content
79 + assert message.role == "assistant"
vllm/entrypoints/openai/serving_chat.py
+2 -2
@@ -361,7 +361,7 @@ class OpenAIServingChat(OpenAIServing):
361
362 # Send response to echo the input portion of the
363 # last message
364 - if request.echo or request.continue_final_message:
364 + if request.echo:
365 last_msg_content: Union[str, List[Dict[str, str]]] = ""
366 if conversation and "content" in conversation[
367 -1] and conversation[-1].get("role") == role:
@@ -706,7 +706,7 @@ class OpenAIServingChat(OpenAIServing):
706 stop_reason=output.stop_reason)
707 choices.append(choice_data)
708
709 - if request.echo or request.continue_final_message:
709 + if request.echo:
710 last_msg_content: Union[str, List[Dict[str, str]]] = ""
711 if conversation and "content" in conversation[-1] and conversation[
712 -1].get("role") == role: