@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 33bab41060
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Bugfix]: Make chat content text allow type content (#9358)

Signed-off-by: Vinay Damodaran <vrdn@hey.com>

Vinay R Damodaran committed Oct 24, 2024 at 01:05 UTC 33bab4106011b4c4b4b68640676a076a2bcccfed
8 files changed +107 -12
docs/source/serving/openai_compatible_server.md
+17
@@ -103,6 +103,23 @@ vllm serve <model> --chat-template ./path-to-chat-template.jinja
103 vLLM community provides a set of chat templates for popular models. You can find them in the examples
104 directory [here](https://github.com/vllm-project/vllm/tree/main/examples/)
105
106 +With the inclusion of multi-modal chat APIs, the OpenAI spec now accepts chat messages in a new format which specifies
107 +both a `type` and a `text` field. An example is provided below:
108 +```python
109 +completion = client.chat.completions.create(
110 + model="NousResearch/Meta-Llama-3-8B-Instruct",
111 + messages=[
112 + {"role": "user", "content": [{"type": "text", "text": "Classify this sentiment: vLLM is wonderful!"}]}
113 + ]
114 +)
115 +```
116 +Most chat templates for LLMs expect the `content` to be a `string` but there are some newer models like
117 +`meta-llama/Llama-Guard-3-1B` that expect the content to be parsed with the new OpenAI spec. In order to choose which
118 +format the content needs to be parsed in by vLLM, please use the `--chat-template-text-format` argument to specify
119 +between `string` or `openai`. The default value is `string` and vLLM internally converts both spec formats to match
120 +this, unless explicitly specified.
121 +
122 +
123 ## Command line arguments for the server
124
125 ```{argparse}
tests/entrypoints/openai/test_serving_chat.py
+1
@@ -26,6 +26,7 @@ class MockModelConfig:
26 tokenizer = MODEL_NAME
27 trust_remote_code = False
28 tokenizer_mode = "auto"
29 + chat_template_text_format = "string"
30 max_model_len = 100
31 tokenizer_revision = None
32 multimodal_config = MultiModalConfig()
tests/entrypoints/test_chat_utils.py
+47 -1
@@ -17,7 +17,7 @@ PHI3V_MODEL_ID = "microsoft/Phi-3.5-vision-instruct"
17 MLLAMA_MODEL_ID = "meta-llama/Llama-3.2-11B-Vision-Instruct"
18
19
20 -@pytest.fixture(scope="module")
20 +@pytest.fixture(scope="function")
21 def phi3v_model_config():
22 return ModelConfig(PHI3V_MODEL_ID,
23 task="generate",
@@ -26,6 +26,7 @@ def phi3v_model_config():
26 trust_remote_code=True,
27 dtype="bfloat16",
28 seed=0,
29 + chat_template_text_format="string",
30 limit_mm_per_prompt={
31 "image": 2,
32 })
@@ -330,6 +331,51 @@ def test_parse_chat_messages_multiple_images_across_messages(
331 _assert_mm_data_is_image_input(mm_data, 2)
332
333
334 +def test_parse_chat_messages_context_text_format(
335 + phi3v_model_config,
336 + phi3v_tokenizer,
337 +):
338 + phi3v_model_config.chat_template_text_format = "openai"
339 + conversation, mm_data = parse_chat_messages(
340 + [{
341 + "role": "user",
342 + "content": [{
343 + "type": "text",
344 + "text": "What's in this text?"
345 + }]
346 + }, {
347 + "role": "assistant",
348 + "content": "Some stuff."
349 + }, {
350 + "role": "user",
351 + "content": "What about this one?"
352 + }], phi3v_model_config, phi3v_tokenizer)
353 +
354 + assert conversation == [
355 + {
356 + "role": "user",
357 + "content": [{
358 + "type": "text",
359 + "text": "What's in this text?"
360 + }]
361 + },
362 + {
363 + "role": "assistant",
364 + "content": [{
365 + "type": "text",
366 + "text": "Some stuff."
367 + }]
368 + },
369 + {
370 + "role": "user",
371 + "content": [{
372 + "type": "text",
373 + "text": "What about this one?"
374 + }]
375 + },
376 + ]
377 +
378 +
379 def test_parse_chat_messages_rejects_too_many_images_in_one_message(
380 phi3v_model_config,
381 phi3v_tokenizer,
vllm/config.py
+2
@@ -142,6 +142,7 @@ class ModelConfig:
142 use_async_output_proc: bool = True,
143 override_neuron_config: Optional[Dict[str, Any]] = None,
144 config_format: ConfigFormat = ConfigFormat.AUTO,
145 + chat_template_text_format: str = "string",
146 mm_processor_kwargs: Optional[Dict[str, Any]] = None) -> None:
147 self.model = model
148 self.tokenizer = tokenizer
@@ -176,6 +177,7 @@ class ModelConfig:
177 self.model, revision)
178 self.dtype = _get_and_verify_dtype(self.hf_text_config, dtype)
179 self.use_async_output_proc = use_async_output_proc
180 + self.chat_template_text_format = chat_template_text_format
181 self.mm_processor_kwargs = mm_processor_kwargs
182
183 # Set enforce_eager to False if the value is unset.
vllm/engine/arg_utils.py
+10
@@ -89,6 +89,7 @@ class EngineArgs:
89 task: TaskOption = "auto"
90 skip_tokenizer_init: bool = False
91 tokenizer_mode: str = 'auto'
92 + chat_template_text_format: str = 'string'
93 trust_remote_code: bool = False
94 download_dir: Optional[str] = None
95 load_format: str = 'auto'
@@ -250,6 +251,14 @@ class EngineArgs:
251 'fast tokenizer if available.\n* "slow" will '
252 'always use the slow tokenizer. \n* '
253 '"mistral" will always use the `mistral_common` tokenizer.')
254 + parser.add_argument(
255 + '--chat-template-text-format',
256 + type=str,
257 + default=EngineArgs.chat_template_text_format,
258 + choices=['string', 'openai'],
259 + help='The format to render text content within a chat template. '
260 + '"string" will keep the content field as a string whereas '
261 + '"openai" will parse content in the current OpenAI format.')
262 parser.add_argument('--trust-remote-code',
263 action='store_true',
264 help='Trust remote code from huggingface.')
@@ -858,6 +867,7 @@ class EngineArgs:
867 # We know this is not None because we set it in __post_init__
868 tokenizer=cast(str, self.tokenizer),
869 tokenizer_mode=self.tokenizer_mode,
870 + chat_template_text_format=self.chat_template_text_format,
871 trust_remote_code=self.trust_remote_code,
872 dtype=self.dtype,
873 seed=self.seed,
vllm/engine/llm_engine.py
+2 -1
@@ -254,7 +254,7 @@ class LLMEngine:
254 "num_scheduler_steps=%d, chunked_prefill_enabled=%s "
255 "multi_step_stream_outputs=%s, enable_prefix_caching=%s, "
256 "use_async_output_proc=%s, use_cached_outputs=%s, "
257 - "mm_processor_kwargs=%s)",
257 + "chat_template_text_format=%s, mm_processor_kwargs=%s)",
258 VLLM_VERSION,
259 model_config.model,
260 speculative_config,
@@ -289,6 +289,7 @@ class LLMEngine:
289 cache_config.enable_prefix_caching,
290 model_config.use_async_output_proc,
291 use_cached_outputs,
292 + model_config.chat_template_text_format,
293 model_config.mm_processor_kwargs,
294 )
295 # TODO(woosuk): Print more configs in debug mode.
vllm/entrypoints/chat_utils.py
+23 -8
@@ -121,7 +121,7 @@ class ConversationMessage(TypedDict, total=False):
121 role: Required[str]
122 """The role of the message's author."""
123
124 - content: Optional[str]
124 + content: Union[Optional[str], List[Dict[str, str]]]
125 """The contents of the message"""
126
127 tool_call_id: Optional[str]
@@ -431,7 +431,7 @@ MM_PARSER_MAP: Dict[str, Callable[[ChatCompletionContentPartParam], str]] = {
431 def _parse_chat_message_content_mm_part(
432 part: ChatCompletionContentPartParam) -> Tuple[str, str]:
433 """
434 - Parses a given multi modal content part based on its type.
434 + Parses a given multi-modal content part based on its type.
435
436 Args:
437 part: A dict containing the content part, with a potential 'type' field.
@@ -485,21 +485,26 @@ def _parse_chat_message_content_parts(
485 role: str,
486 parts: Iterable[ChatCompletionContentPartParam],
487 mm_tracker: BaseMultiModalItemTracker,
488 + chat_template_text_format: str,
489 ) -> List[ConversationMessage]:
490 content: List[Union[str, Dict[str, str]]] = []
491
492 mm_parser = mm_tracker.create_parser()
492 - keep_multimodal_content = \
493 + wrap_dicts = \
494 mm_tracker._model_config.hf_config.model_type in \
494 - MODEL_KEEP_MULTI_MODAL_CONTENT
495 + MODEL_KEEP_MULTI_MODAL_CONTENT or \
496 + (chat_template_text_format == "openai")
497
498 for part in parts:
499 parse_res = _parse_chat_message_content_part(
498 - part, mm_parser, wrap_dicts=keep_multimodal_content)
500 + part,
501 + mm_parser,
502 + wrap_dicts=wrap_dicts,
503 + )
504 if parse_res:
505 content.append(parse_res)
506
502 - if keep_multimodal_content:
507 + if wrap_dicts:
508 # Parsing wraps images and texts as interleaved dictionaries
509 return [ConversationMessage(role=role,
510 content=content)] # type: ignore
@@ -560,6 +565,7 @@ _ToolParser = partial(cast, ChatCompletionToolMessageParam)
565 def _parse_chat_message_content(
566 message: ChatCompletionMessageParam,
567 mm_tracker: BaseMultiModalItemTracker,
568 + chat_template_text_format: str,
569 ) -> List[ConversationMessage]:
570 role = message["role"]
571 content = message.get("content")
@@ -575,6 +581,7 @@ def _parse_chat_message_content(
581 role,
582 content, # type: ignore
583 mm_tracker,
584 + chat_template_text_format,
585 )
586
587 for result_msg in result:
@@ -618,7 +625,11 @@ def parse_chat_messages(
625 mm_tracker = MultiModalItemTracker(model_config, tokenizer)
626
627 for msg in messages:
621 - sub_messages = _parse_chat_message_content(msg, mm_tracker)
628 + sub_messages = _parse_chat_message_content(
629 + msg,
630 + mm_tracker,
631 + model_config.chat_template_text_format,
632 + )
633
634 conversation.extend(sub_messages)
635
@@ -636,7 +647,11 @@ def parse_chat_messages_futures(
647 mm_tracker = AsyncMultiModalItemTracker(model_config, tokenizer)
648
649 for msg in messages:
639 - sub_messages = _parse_chat_message_content(msg, mm_tracker)
650 + sub_messages = _parse_chat_message_content(
651 + msg,
652 + mm_tracker,
653 + model_config.chat_template_text_format,
654 + )
655
656 conversation.extend(sub_messages)
657
vllm/entrypoints/openai/serving_chat.py
+5 -2
@@ -384,7 +384,7 @@ class OpenAIServingChat(OpenAIServing):
384 # Send response to echo the input portion of the
385 # last message
386 if request.echo or request.continue_final_message:
387 - last_msg_content: str = ""
387 + last_msg_content: Union[str, List[Dict[str, str]]] = ""
388 if conversation and "content" in conversation[
389 -1] and conversation[-1].get("role") == role:
390 last_msg_content = conversation[-1]["content"] or ""
@@ -724,10 +724,13 @@ class OpenAIServingChat(OpenAIServing):
724 choices.append(choice_data)
725
726 if request.echo or request.continue_final_message:
727 - last_msg_content = ""
727 + last_msg_content: Union[str, List[Dict[str, str]]] = ""
728 if conversation and "content" in conversation[-1] and conversation[
729 -1].get("role") == role:
730 last_msg_content = conversation[-1]["content"] or ""
731 + if isinstance(last_msg_content, list):
732 + last_msg_content = "\n".join(msg['text']
733 + for msg in last_msg_content)
734
735 for choice in choices:
736 full_message = last_msg_content + (choice.message.content