A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Frontend] Factor out chat message parsing (#7055)
Cyrus Leung committed
Aug 3, 2024 at 12:31 UTC
8c025fa7030350a81bfeb665c99ad622667bdac0
3 files changed
+39
-27
vllm/entrypoints/chat_utils.py
+23
-5
@@ -1,7 +1,8 @@
1
import codecs
2
-from dataclasses import dataclass, field
2
+from dataclasses import dataclass
3
from functools import lru_cache
4
-from typing import Awaitable, Iterable, List, Optional, Union, cast, final
4
+from typing import (Awaitable, Iterable, List, Optional, Tuple, Union, cast,
5
+ final)
6
7
# yapf conflicts with isort for this block
8
# yapf: disable
@@ -65,8 +66,7 @@ class ConversationMessage(TypedDict):
66
@dataclass(frozen=True)
67
class ChatMessageParseResult:
68
messages: List[ConversationMessage]
68
- mm_futures: List[Awaitable[MultiModalDataDict]] = field(
69
- default_factory=list)
69
+ mm_futures: List[Awaitable[MultiModalDataDict]]
70
71
72
def load_chat_template(chat_template: Optional[str]) -> Optional[str]:
@@ -174,7 +174,7 @@ def _parse_chat_message_content_parts(
174
return ChatMessageParseResult(messages=messages, mm_futures=mm_futures)
175
176
177
-def parse_chat_message_content(
177
+def _parse_chat_message_content(
178
message: ChatCompletionMessageParam,
179
model_config: ModelConfig,
180
tokenizer: PreTrainedTokenizer,
@@ -190,3 +190,21 @@ def parse_chat_message_content(
190
191
return _parse_chat_message_content_parts(role, content, model_config,
192
tokenizer)
193
+
194
+
195
+def parse_chat_messages(
196
+ messages: List[ChatCompletionMessageParam],
197
+ model_config: ModelConfig,
198
+ tokenizer: PreTrainedTokenizer,
199
+) -> Tuple[List[ConversationMessage], List[Awaitable[MultiModalDataDict]]]:
200
+ conversation: List[ConversationMessage] = []
201
+ mm_futures: List[Awaitable[MultiModalDataDict]] = []
202
+
203
+ for msg in messages:
204
+ parse_result = _parse_chat_message_content(msg, model_config,
205
+ tokenizer)
206
+
207
+ conversation.extend(parse_result.messages)
208
+ mm_futures.extend(parse_result.mm_futures)
209
+
210
+ return conversation, mm_futures
vllm/entrypoints/openai/serving_chat.py
+5
-12
@@ -1,6 +1,5 @@
1
import time
2
-from typing import (AsyncGenerator, AsyncIterator, Awaitable, Dict, List,
3
- Optional)
2
+from typing import AsyncGenerator, AsyncIterator, Dict, List, Optional
3
from typing import Sequence as GenericSequence
4
from typing import Union
5
@@ -11,7 +10,7 @@ from vllm.config import ModelConfig
10
from vllm.engine.protocol import AsyncEngineClient
11
from vllm.entrypoints.chat_utils import (ConversationMessage,
12
load_chat_template,
14
- parse_chat_message_content)
13
+ parse_chat_messages)
14
from vllm.entrypoints.logger import RequestLogger
15
from vllm.entrypoints.openai.protocol import (
16
ChatCompletionLogProb, ChatCompletionLogProbs,
@@ -92,15 +91,8 @@ class OpenAIServingChat(OpenAIServing):
91
tokenizer = await self.async_engine_client.get_tokenizer(
92
lora_request)
93
95
- conversation: List[ConversationMessage] = []
96
- mm_futures: List[Awaitable[MultiModalDataDict]] = []
97
-
98
- for msg in request.messages:
99
- chat_parsed_result = parse_chat_message_content(
100
- msg, model_config, tokenizer)
101
-
102
- conversation.extend(chat_parsed_result.messages)
103
- mm_futures.extend(chat_parsed_result.mm_futures)
94
+ conversation, mm_futures = parse_chat_messages(
95
+ request.messages, model_config, tokenizer)
96
97
tool_dicts = None if request.tools is None else [
98
tool.model_dump() for tool in request.tools
@@ -115,6 +107,7 @@ class OpenAIServingChat(OpenAIServing):
107
chat_template=request.chat_template or self.chat_template,
108
**(request.chat_template_kwargs or {}),
109
)
110
+ assert isinstance(prompt, str)
111
except Exception as e:
112
logger.error("Error in applying chat template from request: %s", e)
113
return self.create_error_response(str(e))
vllm/entrypoints/openai/serving_tokenization.py
+11
-10
@@ -1,13 +1,11 @@
1
from typing import List, Optional, Union
2
3
from vllm.config import ModelConfig
4
-# yapf conflicts with isort for this block
5
-# yapf: disable
4
from vllm.engine.protocol import AsyncEngineClient
7
-from vllm.entrypoints.chat_utils import (ConversationMessage,
8
- load_chat_template,
9
- parse_chat_message_content)
5
+from vllm.entrypoints.chat_utils import load_chat_template, parse_chat_messages
6
from vllm.entrypoints.logger import RequestLogger
7
+# yapf conflicts with isort for this block
8
+# yapf: disable
9
from vllm.entrypoints.openai.protocol import (DetokenizeRequest,
10
DetokenizeResponse,
11
ErrorResponse,
@@ -17,8 +15,11 @@ from vllm.entrypoints.openai.protocol import (DetokenizeRequest,
15
# yapf: enable
16
from vllm.entrypoints.openai.serving_engine import (LoRAModulePath,
17
OpenAIServing)
18
+from vllm.logger import init_logger
19
from vllm.utils import random_uuid
20
21
+logger = init_logger(__name__)
22
+
23
24
class OpenAIServingTokenization(OpenAIServing):
25
@@ -62,12 +63,12 @@ class OpenAIServingTokenization(OpenAIServing):
63
if isinstance(request, TokenizeChatRequest):
64
model_config = self.model_config
65
65
- conversation: List[ConversationMessage] = []
66
+ conversation, mm_futures = parse_chat_messages(
67
+ request.messages, model_config, tokenizer)
68
67
- for message in request.messages:
68
- result = parse_chat_message_content(message, model_config,
69
- tokenizer)
70
- conversation.extend(result.messages)
69
+ if mm_futures:
70
+ logger.warning(
71
+ "Multi-modal inputs are ignored during tokenization")
72
73
prompt = tokenizer.apply_chat_template(
74
add_generation_prompt=request.add_generation_prompt,