A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Frontend] Move chat utils (#6602)
Co-authored-by: Roger Wang <ywang@roblox.com>
Cyrus Leung committed
Jul 21, 2024 at 08:38 UTC
d7f4178dd99cdba8735d268ff470929de53ee2d5
5 files changed
+54
-49
tests/async_engine/test_chat_template.py
+1
-1
@@ -3,7 +3,7 @@ import pathlib
3
4
import pytest
5
6
-from vllm.entrypoints.openai.chat_utils import load_chat_template
6
+from vllm.entrypoints.chat_utils import load_chat_template
7
from vllm.entrypoints.openai.protocol import ChatCompletionRequest
8
from vllm.transformers_utils.tokenizer import get_tokenizer
9
vllm/entrypoints/chat_utils.py
renamed
+45
-6
@@ -1,15 +1,23 @@
1
import codecs
2
from dataclasses import dataclass, field
3
from functools import lru_cache
4
-from typing import Awaitable, Iterable, List, Optional, TypedDict, cast, final
5
-
6
-from openai.types.chat import (ChatCompletionContentPartImageParam,
7
- ChatCompletionContentPartTextParam)
4
+from typing import Awaitable, Iterable, List, Optional, Union, cast, final
5
+
6
+# yapf conflicts with isort for this block
7
+# yapf: disable
8
+from openai.types.chat import ChatCompletionContentPartImageParam
9
+from openai.types.chat import (
10
+ ChatCompletionContentPartParam as OpenAIChatCompletionContentPartParam)
11
+from openai.types.chat import ChatCompletionContentPartTextParam
12
+from openai.types.chat import (
13
+ ChatCompletionMessageParam as OpenAIChatCompletionMessageParam)
14
+# yapf: enable
15
+# pydantic needs the TypedDict from typing_extensions
16
+from pydantic import ConfigDict
17
from transformers import PreTrainedTokenizer
18
+from typing_extensions import Required, TypedDict
19
20
from vllm.config import ModelConfig
11
-from vllm.entrypoints.openai.protocol import (ChatCompletionContentPartParam,
12
- ChatCompletionMessageParam)
21
from vllm.logger import init_logger
22
from vllm.multimodal import MultiModalDataDict
23
from vllm.multimodal.utils import async_get_and_parse_image
@@ -17,6 +25,37 @@ from vllm.multimodal.utils import async_get_and_parse_image
25
logger = init_logger(__name__)
26
27
28
+class CustomChatCompletionContentPartParam(TypedDict, total=False):
29
+ __pydantic_config__ = ConfigDict(extra="allow") # type: ignore
30
+
31
+ type: Required[str]
32
+ """The type of the content part."""
33
+
34
+
35
+ChatCompletionContentPartParam = Union[OpenAIChatCompletionContentPartParam,
36
+ CustomChatCompletionContentPartParam]
37
+
38
+
39
+class CustomChatCompletionMessageParam(TypedDict, total=False):
40
+ """Enables custom roles in the Chat Completion API."""
41
+ role: Required[str]
42
+ """The role of the message's author."""
43
+
44
+ content: Union[str, List[ChatCompletionContentPartParam]]
45
+ """The contents of the message."""
46
+
47
+ name: str
48
+ """An optional name for the participant.
49
+
50
+ Provides the model information to differentiate between participants of the
51
+ same role.
52
+ """
53
+
54
+
55
+ChatCompletionMessageParam = Union[OpenAIChatCompletionMessageParam,
56
+ CustomChatCompletionMessageParam]
57
+
58
+
59
@final # So that it should be compatible with Dict[str, str]
60
class ConversationMessage(TypedDict):
61
role: str
vllm/entrypoints/openai/protocol.py
+2
-36
@@ -3,50 +3,16 @@
3
import time
4
from typing import Any, Dict, List, Literal, Optional, Union
5
6
-import openai.types.chat
6
import torch
7
from pydantic import BaseModel, ConfigDict, Field, model_validator
9
-# pydantic needs the TypedDict from typing_extensions
10
-from typing_extensions import Annotated, Required, TypedDict
8
+from typing_extensions import Annotated
9
10
+from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
11
from vllm.pooling_params import PoolingParams
12
from vllm.sampling_params import SamplingParams
13
from vllm.utils import random_uuid
14
15
17
-class CustomChatCompletionContentPartParam(TypedDict, total=False):
18
- __pydantic_config__ = ConfigDict(extra="allow") # type: ignore
19
-
20
- type: Required[str]
21
- """The type of the content part."""
22
-
23
-
24
-ChatCompletionContentPartParam = Union[
25
- openai.types.chat.ChatCompletionContentPartParam,
26
- CustomChatCompletionContentPartParam]
27
-
28
-
29
-class CustomChatCompletionMessageParam(TypedDict, total=False):
30
- """Enables custom roles in the Chat Completion API."""
31
- role: Required[str]
32
- """The role of the message's author."""
33
-
34
- content: Union[str, List[ChatCompletionContentPartParam]]
35
- """The contents of the message."""
36
-
37
- name: str
38
- """An optional name for the participant.
39
-
40
- Provides the model information to differentiate between participants of the
41
- same role.
42
- """
43
-
44
-
45
-ChatCompletionMessageParam = Union[
46
- openai.types.chat.ChatCompletionMessageParam,
47
- CustomChatCompletionMessageParam]
48
-
49
-
16
class OpenAIBaseModel(BaseModel):
17
# OpenAI API does not allow extra fields
18
model_config = ConfigDict(extra="forbid")
vllm/entrypoints/openai/serving_chat.py
+3
-3
@@ -9,9 +9,9 @@ from transformers import PreTrainedTokenizer
9
10
from vllm.config import ModelConfig
11
from vllm.engine.async_llm_engine import AsyncLLMEngine
12
-from vllm.entrypoints.openai.chat_utils import (ConversationMessage,
13
- load_chat_template,
14
- parse_chat_message_content)
12
+from vllm.entrypoints.chat_utils import (ConversationMessage,
13
+ load_chat_template,
14
+ parse_chat_message_content)
15
from vllm.entrypoints.openai.protocol import (
16
ChatCompletionLogProb, ChatCompletionLogProbs,
17
ChatCompletionLogProbsContent, ChatCompletionNamedToolChoiceParam,
vllm/entrypoints/openai/serving_tokenization.py
+3
-3
@@ -2,9 +2,9 @@ from typing import List, Optional
2
3
from vllm.config import ModelConfig
4
from vllm.engine.async_llm_engine import AsyncLLMEngine
5
-from vllm.entrypoints.openai.chat_utils import (ConversationMessage,
6
- load_chat_template,
7
- parse_chat_message_content)
5
+from vllm.entrypoints.chat_utils import (ConversationMessage,
6
+ load_chat_template,
7
+ parse_chat_message_content)
8
from vllm.entrypoints.openai.protocol import (DetokenizeRequest,
9
DetokenizeResponse,
10
TokenizeRequest,