@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6c9ba48fde
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Frontend] Added support for HF's new `continue_final_message` parameter (#8942)

danieljannai21 committed Sep 29, 2024 at 20:59 UTC 6c9ba48fdebe2f44c82eabfe136dc8dc6ad6f4ed
7 files changed +105 -31
tests/entrypoints/openai/test_chat_template.py
+23 -7
@@ -12,7 +12,7 @@ assert chatml_jinja_path.exists()
12
13 # Define models, templates, and their corresponding expected outputs
14 MODEL_TEMPLATE_GENERATON_OUTPUT = [
15 - ("facebook/opt-125m", chatml_jinja_path, True, """<|im_start|>user
15 + ("facebook/opt-125m", chatml_jinja_path, True, False, """<|im_start|>user
16 Hello<|im_end|>
17 <|im_start|>assistant
18 Hi there!<|im_end|>
@@ -20,12 +20,20 @@ Hi there!<|im_end|>
20 What is the capital of<|im_end|>
21 <|im_start|>assistant
22 """),
23 - ("facebook/opt-125m", chatml_jinja_path, False, """<|im_start|>user
23 + ("facebook/opt-125m", chatml_jinja_path, False, False, """<|im_start|>user
24 Hello<|im_end|>
25 <|im_start|>assistant
26 Hi there!<|im_end|>
27 <|im_start|>user
28 -What is the capital of""")
28 +What is the capital of"""),
29 + ("facebook/opt-125m", chatml_jinja_path, False, True, """<|im_start|>user
30 +Hello<|im_end|>
31 +<|im_start|>assistant
32 +Hi there!<|im_end|>
33 +<|im_start|>user
34 +What is the capital of<|im_end|>
35 +<|im_start|>assistant
36 +The capital of"""),
37 ]
38
39 TEST_MESSAGES = [
@@ -42,6 +50,10 @@ TEST_MESSAGES = [
50 'content': 'What is the capital of'
51 },
52 ]
53 +ASSISTANT_MESSAGE_TO_CONTINUE = {
54 + 'role': 'assistant',
55 + 'content': 'The capital of'
56 +}
57
58
59 def test_load_chat_template():
@@ -73,10 +85,10 @@ def test_no_load_chat_template_literallike():
85
86
87 @pytest.mark.parametrize(
76 - "model,template,add_generation_prompt,expected_output",
88 + "model,template,add_generation_prompt,continue_final_message,expected_output",
89 MODEL_TEMPLATE_GENERATON_OUTPUT)
90 def test_get_gen_prompt(model, template, add_generation_prompt,
79 - expected_output):
91 + continue_final_message, expected_output):
92 # Initialize the tokenizer
93 tokenizer = get_tokenizer(tokenizer_name=model)
94 template_content = load_chat_template(chat_template=template)
@@ -84,8 +96,11 @@ def test_get_gen_prompt(model, template, add_generation_prompt,
96 # Create a mock request object using keyword arguments
97 mock_request = ChatCompletionRequest(
98 model=model,
87 - messages=TEST_MESSAGES,
88 - add_generation_prompt=add_generation_prompt)
99 + messages=TEST_MESSAGES + [ASSISTANT_MESSAGE_TO_CONTINUE]
100 + if continue_final_message else TEST_MESSAGES,
101 + add_generation_prompt=add_generation_prompt,
102 + continue_final_message=continue_final_message,
103 + )
104
105 # Call the function and get the result
106 result = apply_hf_chat_template(
@@ -93,6 +108,7 @@ def test_get_gen_prompt(model, template, add_generation_prompt,
108 conversation=mock_request.messages,
109 chat_template=mock_request.chat_template or template_content,
110 add_generation_prompt=mock_request.add_generation_prompt,
111 + continue_final_message=mock_request.continue_final_message,
112 )
113
114 # Test assertion
tests/entrypoints/openai/test_tokenization.py
+34 -22
@@ -104,28 +104,40 @@ async def test_tokenize_chat(client: openai.AsyncOpenAI, model_name: str,
104 "role": "user",
105 "content": "Can I ask a question? vllm1"
106 }]
107 -
108 - prompt = tokenizer.apply_chat_template(
109 - add_generation_prompt=add_generation,
110 - conversation=conversation,
111 - tokenize=False)
112 - tokens = tokenizer.encode(prompt, add_special_tokens=add_special)
113 -
114 - response = requests.post(base_url + "/tokenize",
115 - json={
116 - "add_generation_prompt":
117 - add_generation,
118 - "add_special_tokens": add_special,
119 - "messages": conversation,
120 - "model": model_name
121 - })
122 - response.raise_for_status()
123 -
124 - assert response.json() == {
125 - "tokens": tokens,
126 - "count": len(tokens),
127 - "max_model_len": 8192
128 - }
107 + for continue_final in [False, True]:
108 + if add_generation and continue_final:
109 + continue
110 + if continue_final:
111 + conversation.append({
112 + "role": "assistant",
113 + "content": "Sure,"
114 + })
115 +
116 + prompt = tokenizer.apply_chat_template(
117 + add_generation_prompt=add_generation,
118 + continue_final_message=continue_final,
119 + conversation=conversation,
120 + tokenize=False)
121 + tokens = tokenizer.encode(prompt,
122 + add_special_tokens=add_special)
123 +
124 + response = requests.post(base_url + "/tokenize",
125 + json={
126 + "add_generation_prompt":
127 + add_generation,
128 + "continue_final_message":
129 + continue_final,
130 + "add_special_tokens": add_special,
131 + "messages": conversation,
132 + "model": model_name
133 + })
134 + response.raise_for_status()
135 +
136 + assert response.json() == {
137 + "tokens": tokens,
138 + "count": len(tokens),
139 + "max_model_len": 8192
140 + }
141
142
143 @pytest.mark.asyncio
vllm/entrypoints/chat_utils.py
+8
@@ -542,6 +542,14 @@ def apply_mistral_chat_template(
542 if chat_template is not None:
543 logger.warning(
544 "'chat_template' cannot be overridden for mistral tokenizer.")
545 + if "add_generation_prompt" in kwargs:
546 + logger.warning(
547 + "'add_generation_prompt' is not supported for mistral tokenizer, "
548 + "so it will be ignored.")
549 + if "continue_final_message" in kwargs:
550 + logger.warning(
551 + "'continue_final_message' is not supported for mistral tokenizer, "
552 + "so it will be ignored.")
553
554 return tokenizer.apply_chat_template(
555 messages=messages,
vllm/entrypoints/llm.py
+6
@@ -501,6 +501,7 @@ class LLM:
501 lora_request: Optional[LoRARequest] = None,
502 chat_template: Optional[str] = None,
503 add_generation_prompt: bool = True,
504 + continue_final_message: bool = False,
505 tools: Optional[List[Dict[str, Any]]] = None,
506 ) -> List[RequestOutput]:
507 """
@@ -528,6 +529,9 @@ class LLM:
529 If not provided, the model's default chat template will be used.
530 add_generation_prompt: If True, adds a generation template
531 to each message.
532 + continue_final_message: If True, continues the final message in
533 + the conversation instead of starting a new one. Cannot be `True`
534 + if `add_generation_prompt` is also `True`.
535
536 Returns:
537 A list of ``RequestOutput`` objects containing the generated
@@ -559,6 +563,7 @@ class LLM:
563 messages=msgs,
564 chat_template=chat_template,
565 add_generation_prompt=add_generation_prompt,
566 + continue_final_message=continue_final_message,
567 tools=tools,
568 )
569 else:
@@ -567,6 +572,7 @@ class LLM:
572 conversation=conversation,
573 chat_template=chat_template,
574 add_generation_prompt=add_generation_prompt,
575 + continue_final_message=continue_final_message,
576 tools=tools,
577 )
578
vllm/entrypoints/openai/protocol.py
+28
@@ -211,6 +211,15 @@ class ChatCompletionRequest(OpenAIBaseModel):
211 "This is a parameter used by chat template in tokenizer config of the "
212 "model."),
213 )
214 + continue_final_message: bool = Field(
215 + default=False,
216 + description=
217 + ("If this is set, the chat will be formatted so that the final "
218 + "message in the chat is open-ended, without any EOS tokens. The "
219 + "model will continue this message rather than starting a new one. "
220 + "This allows you to \"prefill\" part of the model's response for it. "
221 + "Cannot be used at the same time as `add_generation_prompt`."),
222 + )
223 add_special_tokens: bool = Field(
224 default=False,
225 description=(
@@ -431,6 +440,15 @@ class ChatCompletionRequest(OpenAIBaseModel):
440 " of the specified `tools`")
441 return data
442
443 + @model_validator(mode="before")
444 + @classmethod
445 + def check_generation_prompt(cls, data):
446 + if data.get("continue_final_message") and data.get(
447 + "add_generation_prompt"):
448 + raise ValueError("Cannot set both `continue_final_message` and "
449 + "`add_generation_prompt` to True.")
450 + return data
451 +
452
453 class CompletionRequest(OpenAIBaseModel):
454 # Ordered by official OpenAI API documentation
@@ -862,8 +880,18 @@ class TokenizeChatRequest(OpenAIBaseModel):
880 messages: List[ChatCompletionMessageParam]
881
882 add_generation_prompt: bool = Field(default=True)
883 + continue_final_message: bool = Field(default=False)
884 add_special_tokens: bool = Field(default=False)
885
886 + @model_validator(mode="before")
887 + @classmethod
888 + def check_generation_prompt(cls, data):
889 + if data.get("continue_final_message") and data.get(
890 + "add_generation_prompt"):
891 + raise ValueError("Cannot set both `continue_final_message` and "
892 + "`add_generation_prompt` to True.")
893 + return data
894 +
895
896 TokenizeRequest = Union[TokenizeCompletionRequest, TokenizeChatRequest]
897
vllm/entrypoints/openai/serving_chat.py
+4 -2
@@ -140,6 +140,7 @@ class OpenAIServingChat(OpenAIServing):
140 messages=request.messages,
141 chat_template=request.chat_template or self.chat_template,
142 add_generation_prompt=request.add_generation_prompt,
143 + continue_final_message=request.continue_final_message,
144 tools=tool_dicts,
145 documents=request.documents,
146 **(request.chat_template_kwargs or {}),
@@ -150,6 +151,7 @@ class OpenAIServingChat(OpenAIServing):
151 conversation=conversation,
152 chat_template=request.chat_template or self.chat_template,
153 add_generation_prompt=request.add_generation_prompt,
154 + continue_final_message=request.continue_final_message,
155 tools=tool_dicts,
156 documents=request.documents,
157 **(request.chat_template_kwargs or {}),
@@ -361,7 +363,7 @@ class OpenAIServingChat(OpenAIServing):
363
364 # Send response to echo the input portion of the
365 # last message
364 - if request.echo:
366 + if request.echo or request.continue_final_message:
367 last_msg_content: str = ""
368 if conversation and "content" in conversation[
369 -1] and conversation[-1].get("role") == role:
@@ -716,7 +718,7 @@ class OpenAIServingChat(OpenAIServing):
718 stop_reason=output.stop_reason)
719 choices.append(choice_data)
720
719 - if request.echo:
721 + if request.echo or request.continue_final_message:
722 last_msg_content = ""
723 if conversation and "content" in conversation[-1] and conversation[
724 -1].get("role") == role:
vllm/entrypoints/openai/serving_tokenization.py
+2
@@ -87,6 +87,7 @@ class OpenAIServingTokenization(OpenAIServing):
87 messages=request.messages,
88 chat_template=self.chat_template,
89 add_generation_prompt=request.add_generation_prompt,
90 + continue_final_message=request.continue_final_message,
91 )
92 else:
93 prompt = apply_hf_chat_template(
@@ -94,6 +95,7 @@ class OpenAIServingTokenization(OpenAIServing):
95 conversation=conversation,
96 chat_template=self.chat_template,
97 add_generation_prompt=request.add_generation_prompt,
98 + continue_final_message=request.continue_final_message,
99 )
100 else:
101 prompt = request.prompt