A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[Bug]: When apply continue_final_message for OpenAI server, the "echo":false is ignored (#10180)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
Chauncey committed
Nov 22, 2024 at 00:24 UTC
da7e702c6fae521bf8633affb8fe7b834f5cb94b
2 files changed
+81
-2
tests/entrypoints/openai/test_chat_echo.py
new
+79
@@ -0,0 +1,79 @@
1
+from typing import NamedTuple
2
+
3
+import openai # use the official client for correctness check
4
+import pytest
5
+import pytest_asyncio
6
+
7
+from ...utils import RemoteOpenAIServer
8
+
9
+# # any model with a chat template should work here
10
+MODEL_NAME = "Qwen/Qwen2-1.5B-Instruct"
11
+DUMMY_CHAT_TEMPLATE = """{% for message in messages %}{{message['role'] + ': ' + message['content'] + '\\n'}}{% endfor %}""" # noqa: E501
12
+
13
+
14
+@pytest.fixture(scope="module")
15
+def server():
16
+ args = [
17
+ # use half precision for speed and memory savings in CI environment
18
+ "--dtype",
19
+ "float16",
20
+ "--enforce-eager",
21
+ "--max-model-len",
22
+ "4080",
23
+ "--chat-template",
24
+ DUMMY_CHAT_TEMPLATE,
25
+ ]
26
+
27
+ with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
28
+ yield remote_server
29
+
30
+
31
+@pytest_asyncio.fixture
32
+async def client(server):
33
+ async with server.get_async_client() as async_client:
34
+ yield async_client
35
+
36
+
37
+class TestCase(NamedTuple):
38
+ model_name: str
39
+ echo: bool
40
+
41
+
42
+@pytest.mark.asyncio
43
+@pytest.mark.parametrize(
44
+ "test_case",
45
+ [
46
+ TestCase(model_name=MODEL_NAME, echo=True),
47
+ TestCase(model_name=MODEL_NAME, echo=False)
48
+ ],
49
+)
50
+async def test_chat_session_with_echo_and_continue_final_message(
51
+ client: openai.AsyncOpenAI, test_case: TestCase):
52
+ saying: str = "Here is a common saying about apple. An apple a day, keeps"
53
+ # test echo with continue_final_message parameter
54
+ chat_completion = await client.chat.completions.create(
55
+ model=test_case.model_name,
56
+ messages=[{
57
+ "role": "user",
58
+ "content": "tell me a common saying"
59
+ }, {
60
+ "role": "assistant",
61
+ "content": saying
62
+ }],
63
+ extra_body={
64
+ "echo": test_case.echo,
65
+ "continue_final_message": True,
66
+ "add_generation_prompt": False
67
+ })
68
+ assert chat_completion.id is not None
69
+ assert len(chat_completion.choices) == 1
70
+
71
+ choice = chat_completion.choices[0]
72
+ assert choice.finish_reason == "stop"
73
+
74
+ message = choice.message
75
+ if test_case.echo:
76
+ assert message.content is not None and saying in message.content
77
+ else:
78
+ assert message.content is not None and saying not in message.content
79
+ assert message.role == "assistant"
vllm/entrypoints/openai/serving_chat.py
+2
-2
@@ -361,7 +361,7 @@ class OpenAIServingChat(OpenAIServing):
361
362
# Send response to echo the input portion of the
363
# last message
364
- if request.echo or request.continue_final_message:
364
+ if request.echo:
365
last_msg_content: Union[str, List[Dict[str, str]]] = ""
366
if conversation and "content" in conversation[
367
-1] and conversation[-1].get("role") == role:
@@ -706,7 +706,7 @@ class OpenAIServingChat(OpenAIServing):
706
stop_reason=output.stop_reason)
707
choices.append(choice_data)
708
709
- if request.echo or request.continue_final_message:
709
+ if request.echo:
710
last_msg_content: Union[str, List[Dict[str, str]]] = ""
711
if conversation and "content" in conversation[-1] and conversation[
712
-1].get("role") == role: