A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Frontend] Added support for HF's new `continue_final_message` parameter (#8942)
danieljannai21 committed
Sep 29, 2024 at 20:59 UTC
6c9ba48fdebe2f44c82eabfe136dc8dc6ad6f4ed
7 files changed
+105
-31
tests/entrypoints/openai/test_chat_template.py
+23
-7
@@ -12,7 +12,7 @@ assert chatml_jinja_path.exists()
12
13
# Define models, templates, and their corresponding expected outputs
14
MODEL_TEMPLATE_GENERATON_OUTPUT = [
15
- ("facebook/opt-125m", chatml_jinja_path, True, """<|im_start|>user
15
+ ("facebook/opt-125m", chatml_jinja_path, True, False, """<|im_start|>user
16
Hello<|im_end|>
17
<|im_start|>assistant
18
Hi there!<|im_end|>
@@ -20,12 +20,20 @@ Hi there!<|im_end|>
20
What is the capital of<|im_end|>
21
<|im_start|>assistant
22
"""),
23
- ("facebook/opt-125m", chatml_jinja_path, False, """<|im_start|>user
23
+ ("facebook/opt-125m", chatml_jinja_path, False, False, """<|im_start|>user
24
Hello<|im_end|>
25
<|im_start|>assistant
26
Hi there!<|im_end|>
27
<|im_start|>user
28
-What is the capital of""")
28
+What is the capital of"""),
29
+ ("facebook/opt-125m", chatml_jinja_path, False, True, """<|im_start|>user
30
+Hello<|im_end|>
31
+<|im_start|>assistant
32
+Hi there!<|im_end|>
33
+<|im_start|>user
34
+What is the capital of<|im_end|>
35
+<|im_start|>assistant
36
+The capital of"""),
37
]
38
39
TEST_MESSAGES = [
@@ -42,6 +50,10 @@ TEST_MESSAGES = [
50
'content': 'What is the capital of'
51
},
52
]
53
+ASSISTANT_MESSAGE_TO_CONTINUE = {
54
+ 'role': 'assistant',
55
+ 'content': 'The capital of'
56
+}
57
58
59
def test_load_chat_template():
@@ -73,10 +85,10 @@ def test_no_load_chat_template_literallike():
85
86
87
@pytest.mark.parametrize(
76
- "model,template,add_generation_prompt,expected_output",
88
+ "model,template,add_generation_prompt,continue_final_message,expected_output",
89
MODEL_TEMPLATE_GENERATON_OUTPUT)
90
def test_get_gen_prompt(model, template, add_generation_prompt,
79
- expected_output):
91
+ continue_final_message, expected_output):
92
# Initialize the tokenizer
93
tokenizer = get_tokenizer(tokenizer_name=model)
94
template_content = load_chat_template(chat_template=template)
@@ -84,8 +96,11 @@ def test_get_gen_prompt(model, template, add_generation_prompt,
96
# Create a mock request object using keyword arguments
97
mock_request = ChatCompletionRequest(
98
model=model,
87
- messages=TEST_MESSAGES,
88
- add_generation_prompt=add_generation_prompt)
99
+ messages=TEST_MESSAGES + [ASSISTANT_MESSAGE_TO_CONTINUE]
100
+ if continue_final_message else TEST_MESSAGES,
101
+ add_generation_prompt=add_generation_prompt,
102
+ continue_final_message=continue_final_message,
103
+ )
104
105
# Call the function and get the result
106
result = apply_hf_chat_template(
@@ -93,6 +108,7 @@ def test_get_gen_prompt(model, template, add_generation_prompt,
108
conversation=mock_request.messages,
109
chat_template=mock_request.chat_template or template_content,
110
add_generation_prompt=mock_request.add_generation_prompt,
111
+ continue_final_message=mock_request.continue_final_message,
112
)
113
114
# Test assertion
tests/entrypoints/openai/test_tokenization.py
+34
-22
@@ -104,28 +104,40 @@ async def test_tokenize_chat(client: openai.AsyncOpenAI, model_name: str,
104
"role": "user",
105
"content": "Can I ask a question? vllm1"
106
}]
107
-
108
- prompt = tokenizer.apply_chat_template(
109
- add_generation_prompt=add_generation,
110
- conversation=conversation,
111
- tokenize=False)
112
- tokens = tokenizer.encode(prompt, add_special_tokens=add_special)
113
-
114
- response = requests.post(base_url + "/tokenize",
115
- json={
116
- "add_generation_prompt":
117
- add_generation,
118
- "add_special_tokens": add_special,
119
- "messages": conversation,
120
- "model": model_name
121
- })
122
- response.raise_for_status()
123
-
124
- assert response.json() == {
125
- "tokens": tokens,
126
- "count": len(tokens),
127
- "max_model_len": 8192
128
- }
107
+ for continue_final in [False, True]:
108
+ if add_generation and continue_final:
109
+ continue
110
+ if continue_final:
111
+ conversation.append({
112
+ "role": "assistant",
113
+ "content": "Sure,"
114
+ })
115
+
116
+ prompt = tokenizer.apply_chat_template(
117
+ add_generation_prompt=add_generation,
118
+ continue_final_message=continue_final,
119
+ conversation=conversation,
120
+ tokenize=False)
121
+ tokens = tokenizer.encode(prompt,
122
+ add_special_tokens=add_special)
123
+
124
+ response = requests.post(base_url + "/tokenize",
125
+ json={
126
+ "add_generation_prompt":
127
+ add_generation,
128
+ "continue_final_message":
129
+ continue_final,
130
+ "add_special_tokens": add_special,
131
+ "messages": conversation,
132
+ "model": model_name
133
+ })
134
+ response.raise_for_status()
135
+
136
+ assert response.json() == {
137
+ "tokens": tokens,
138
+ "count": len(tokens),
139
+ "max_model_len": 8192
140
+ }
141
142
143
@pytest.mark.asyncio
vllm/entrypoints/chat_utils.py
+8
@@ -542,6 +542,14 @@ def apply_mistral_chat_template(
542
if chat_template is not None:
543
logger.warning(
544
"'chat_template' cannot be overridden for mistral tokenizer.")
545
+ if "add_generation_prompt" in kwargs:
546
+ logger.warning(
547
+ "'add_generation_prompt' is not supported for mistral tokenizer, "
548
+ "so it will be ignored.")
549
+ if "continue_final_message" in kwargs:
550
+ logger.warning(
551
+ "'continue_final_message' is not supported for mistral tokenizer, "
552
+ "so it will be ignored.")
553
554
return tokenizer.apply_chat_template(
555
messages=messages,
vllm/entrypoints/llm.py
+6
@@ -501,6 +501,7 @@ class LLM:
501
lora_request: Optional[LoRARequest] = None,
502
chat_template: Optional[str] = None,
503
add_generation_prompt: bool = True,
504
+ continue_final_message: bool = False,
505
tools: Optional[List[Dict[str, Any]]] = None,
506
) -> List[RequestOutput]:
507
"""
@@ -528,6 +529,9 @@ class LLM:
529
If not provided, the model's default chat template will be used.
530
add_generation_prompt: If True, adds a generation template
531
to each message.
532
+ continue_final_message: If True, continues the final message in
533
+ the conversation instead of starting a new one. Cannot be `True`
534
+ if `add_generation_prompt` is also `True`.
535
536
Returns:
537
A list of ``RequestOutput`` objects containing the generated
@@ -559,6 +563,7 @@ class LLM:
563
messages=msgs,
564
chat_template=chat_template,
565
add_generation_prompt=add_generation_prompt,
566
+ continue_final_message=continue_final_message,
567
tools=tools,
568
)
569
else:
@@ -567,6 +572,7 @@ class LLM:
572
conversation=conversation,
573
chat_template=chat_template,
574
add_generation_prompt=add_generation_prompt,
575
+ continue_final_message=continue_final_message,
576
tools=tools,
577
)
578
vllm/entrypoints/openai/protocol.py
+28
@@ -211,6 +211,15 @@ class ChatCompletionRequest(OpenAIBaseModel):
211
"This is a parameter used by chat template in tokenizer config of the "
212
"model."),
213
)
214
+ continue_final_message: bool = Field(
215
+ default=False,
216
+ description=
217
+ ("If this is set, the chat will be formatted so that the final "
218
+ "message in the chat is open-ended, without any EOS tokens. The "
219
+ "model will continue this message rather than starting a new one. "
220
+ "This allows you to \"prefill\" part of the model's response for it. "
221
+ "Cannot be used at the same time as `add_generation_prompt`."),
222
+ )
223
add_special_tokens: bool = Field(
224
default=False,
225
description=(
@@ -431,6 +440,15 @@ class ChatCompletionRequest(OpenAIBaseModel):
440
" of the specified `tools`")
441
return data
442
443
+ @model_validator(mode="before")
444
+ @classmethod
445
+ def check_generation_prompt(cls, data):
446
+ if data.get("continue_final_message") and data.get(
447
+ "add_generation_prompt"):
448
+ raise ValueError("Cannot set both `continue_final_message` and "
449
+ "`add_generation_prompt` to True.")
450
+ return data
451
+
452
453
class CompletionRequest(OpenAIBaseModel):
454
# Ordered by official OpenAI API documentation
@@ -862,8 +880,18 @@ class TokenizeChatRequest(OpenAIBaseModel):
880
messages: List[ChatCompletionMessageParam]
881
882
add_generation_prompt: bool = Field(default=True)
883
+ continue_final_message: bool = Field(default=False)
884
add_special_tokens: bool = Field(default=False)
885
886
+ @model_validator(mode="before")
887
+ @classmethod
888
+ def check_generation_prompt(cls, data):
889
+ if data.get("continue_final_message") and data.get(
890
+ "add_generation_prompt"):
891
+ raise ValueError("Cannot set both `continue_final_message` and "
892
+ "`add_generation_prompt` to True.")
893
+ return data
894
+
895
896
TokenizeRequest = Union[TokenizeCompletionRequest, TokenizeChatRequest]
897
vllm/entrypoints/openai/serving_chat.py
+4
-2
@@ -140,6 +140,7 @@ class OpenAIServingChat(OpenAIServing):
140
messages=request.messages,
141
chat_template=request.chat_template or self.chat_template,
142
add_generation_prompt=request.add_generation_prompt,
143
+ continue_final_message=request.continue_final_message,
144
tools=tool_dicts,
145
documents=request.documents,
146
**(request.chat_template_kwargs or {}),
@@ -150,6 +151,7 @@ class OpenAIServingChat(OpenAIServing):
151
conversation=conversation,
152
chat_template=request.chat_template or self.chat_template,
153
add_generation_prompt=request.add_generation_prompt,
154
+ continue_final_message=request.continue_final_message,
155
tools=tool_dicts,
156
documents=request.documents,
157
**(request.chat_template_kwargs or {}),
@@ -361,7 +363,7 @@ class OpenAIServingChat(OpenAIServing):
363
364
# Send response to echo the input portion of the
365
# last message
364
- if request.echo:
366
+ if request.echo or request.continue_final_message:
367
last_msg_content: str = ""
368
if conversation and "content" in conversation[
369
-1] and conversation[-1].get("role") == role:
@@ -716,7 +718,7 @@ class OpenAIServingChat(OpenAIServing):
718
stop_reason=output.stop_reason)
719
choices.append(choice_data)
720
719
- if request.echo:
721
+ if request.echo or request.continue_final_message:
722
last_msg_content = ""
723
if conversation and "content" in conversation[-1] and conversation[
724
-1].get("role") == role:
vllm/entrypoints/openai/serving_tokenization.py
+2
@@ -87,6 +87,7 @@ class OpenAIServingTokenization(OpenAIServing):
87
messages=request.messages,
88
chat_template=self.chat_template,
89
add_generation_prompt=request.add_generation_prompt,
90
+ continue_final_message=request.continue_final_message,
91
)
92
else:
93
prompt = apply_hf_chat_template(
@@ -94,6 +95,7 @@ class OpenAIServingTokenization(OpenAIServing):
95
conversation=conversation,
96
chat_template=self.chat_template,
97
add_generation_prompt=request.add_generation_prompt,
98
+ continue_final_message=request.continue_final_message,
99
)
100
else:
101
prompt = request.prompt