@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8c025fa703
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Frontend] Factor out chat message parsing (#7055)

Cyrus Leung committed Aug 3, 2024 at 12:31 UTC 8c025fa7030350a81bfeb665c99ad622667bdac0
3 files changed +39 -27
vllm/entrypoints/chat_utils.py
+23 -5
@@ -1,7 +1,8 @@
1 import codecs
2 -from dataclasses import dataclass, field
2 +from dataclasses import dataclass
3 from functools import lru_cache
4 -from typing import Awaitable, Iterable, List, Optional, Union, cast, final
4 +from typing import (Awaitable, Iterable, List, Optional, Tuple, Union, cast,
5 + final)
6
7 # yapf conflicts with isort for this block
8 # yapf: disable
@@ -65,8 +66,7 @@ class ConversationMessage(TypedDict):
66 @dataclass(frozen=True)
67 class ChatMessageParseResult:
68 messages: List[ConversationMessage]
68 - mm_futures: List[Awaitable[MultiModalDataDict]] = field(
69 - default_factory=list)
69 + mm_futures: List[Awaitable[MultiModalDataDict]]
70
71
72 def load_chat_template(chat_template: Optional[str]) -> Optional[str]:
@@ -174,7 +174,7 @@ def _parse_chat_message_content_parts(
174 return ChatMessageParseResult(messages=messages, mm_futures=mm_futures)
175
176
177 -def parse_chat_message_content(
177 +def _parse_chat_message_content(
178 message: ChatCompletionMessageParam,
179 model_config: ModelConfig,
180 tokenizer: PreTrainedTokenizer,
@@ -190,3 +190,21 @@ def parse_chat_message_content(
190
191 return _parse_chat_message_content_parts(role, content, model_config,
192 tokenizer)
193 +
194 +
195 +def parse_chat_messages(
196 + messages: List[ChatCompletionMessageParam],
197 + model_config: ModelConfig,
198 + tokenizer: PreTrainedTokenizer,
199 +) -> Tuple[List[ConversationMessage], List[Awaitable[MultiModalDataDict]]]:
200 + conversation: List[ConversationMessage] = []
201 + mm_futures: List[Awaitable[MultiModalDataDict]] = []
202 +
203 + for msg in messages:
204 + parse_result = _parse_chat_message_content(msg, model_config,
205 + tokenizer)
206 +
207 + conversation.extend(parse_result.messages)
208 + mm_futures.extend(parse_result.mm_futures)
209 +
210 + return conversation, mm_futures
vllm/entrypoints/openai/serving_chat.py
+5 -12
@@ -1,6 +1,5 @@
1 import time
2 -from typing import (AsyncGenerator, AsyncIterator, Awaitable, Dict, List,
3 - Optional)
2 +from typing import AsyncGenerator, AsyncIterator, Dict, List, Optional
3 from typing import Sequence as GenericSequence
4 from typing import Union
5
@@ -11,7 +10,7 @@ from vllm.config import ModelConfig
10 from vllm.engine.protocol import AsyncEngineClient
11 from vllm.entrypoints.chat_utils import (ConversationMessage,
12 load_chat_template,
14 - parse_chat_message_content)
13 + parse_chat_messages)
14 from vllm.entrypoints.logger import RequestLogger
15 from vllm.entrypoints.openai.protocol import (
16 ChatCompletionLogProb, ChatCompletionLogProbs,
@@ -92,15 +91,8 @@ class OpenAIServingChat(OpenAIServing):
91 tokenizer = await self.async_engine_client.get_tokenizer(
92 lora_request)
93
95 - conversation: List[ConversationMessage] = []
96 - mm_futures: List[Awaitable[MultiModalDataDict]] = []
97 -
98 - for msg in request.messages:
99 - chat_parsed_result = parse_chat_message_content(
100 - msg, model_config, tokenizer)
101 -
102 - conversation.extend(chat_parsed_result.messages)
103 - mm_futures.extend(chat_parsed_result.mm_futures)
94 + conversation, mm_futures = parse_chat_messages(
95 + request.messages, model_config, tokenizer)
96
97 tool_dicts = None if request.tools is None else [
98 tool.model_dump() for tool in request.tools
@@ -115,6 +107,7 @@ class OpenAIServingChat(OpenAIServing):
107 chat_template=request.chat_template or self.chat_template,
108 **(request.chat_template_kwargs or {}),
109 )
110 + assert isinstance(prompt, str)
111 except Exception as e:
112 logger.error("Error in applying chat template from request: %s", e)
113 return self.create_error_response(str(e))
vllm/entrypoints/openai/serving_tokenization.py
+11 -10
@@ -1,13 +1,11 @@
1 from typing import List, Optional, Union
2
3 from vllm.config import ModelConfig
4 -# yapf conflicts with isort for this block
5 -# yapf: disable
4 from vllm.engine.protocol import AsyncEngineClient
7 -from vllm.entrypoints.chat_utils import (ConversationMessage,
8 - load_chat_template,
9 - parse_chat_message_content)
5 +from vllm.entrypoints.chat_utils import load_chat_template, parse_chat_messages
6 from vllm.entrypoints.logger import RequestLogger
7 +# yapf conflicts with isort for this block
8 +# yapf: disable
9 from vllm.entrypoints.openai.protocol import (DetokenizeRequest,
10 DetokenizeResponse,
11 ErrorResponse,
@@ -17,8 +15,11 @@ from vllm.entrypoints.openai.protocol import (DetokenizeRequest,
15 # yapf: enable
16 from vllm.entrypoints.openai.serving_engine import (LoRAModulePath,
17 OpenAIServing)
18 +from vllm.logger import init_logger
19 from vllm.utils import random_uuid
20
21 +logger = init_logger(__name__)
22 +
23
24 class OpenAIServingTokenization(OpenAIServing):
25
@@ -62,12 +63,12 @@ class OpenAIServingTokenization(OpenAIServing):
63 if isinstance(request, TokenizeChatRequest):
64 model_config = self.model_config
65
65 - conversation: List[ConversationMessage] = []
66 + conversation, mm_futures = parse_chat_messages(
67 + request.messages, model_config, tokenizer)
68
67 - for message in request.messages:
68 - result = parse_chat_message_content(message, model_config,
69 - tokenizer)
70 - conversation.extend(result.messages)
69 + if mm_futures:
70 + logger.warning(
71 + "Multi-modal inputs are ignored during tokenization")
72
73 prompt = tokenizer.apply_chat_template(
74 add_generation_prompt=request.add_generation_prompt,