@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d7f4178dd9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Frontend] Move chat utils (#6602)

Co-authored-by: Roger Wang <ywang@roblox.com>

Cyrus Leung committed Jul 21, 2024 at 08:38 UTC d7f4178dd99cdba8735d268ff470929de53ee2d5
5 files changed +54 -49
tests/async_engine/test_chat_template.py
+1 -1
@@ -3,7 +3,7 @@ import pathlib
3
4 import pytest
5
6 -from vllm.entrypoints.openai.chat_utils import load_chat_template
6 +from vllm.entrypoints.chat_utils import load_chat_template
7 from vllm.entrypoints.openai.protocol import ChatCompletionRequest
8 from vllm.transformers_utils.tokenizer import get_tokenizer
9
vllm/entrypoints/chat_utils.py renamed
+45 -6
@@ -1,15 +1,23 @@
1 import codecs
2 from dataclasses import dataclass, field
3 from functools import lru_cache
4 -from typing import Awaitable, Iterable, List, Optional, TypedDict, cast, final
5 -
6 -from openai.types.chat import (ChatCompletionContentPartImageParam,
7 - ChatCompletionContentPartTextParam)
4 +from typing import Awaitable, Iterable, List, Optional, Union, cast, final
5 +
6 +# yapf conflicts with isort for this block
7 +# yapf: disable
8 +from openai.types.chat import ChatCompletionContentPartImageParam
9 +from openai.types.chat import (
10 + ChatCompletionContentPartParam as OpenAIChatCompletionContentPartParam)
11 +from openai.types.chat import ChatCompletionContentPartTextParam
12 +from openai.types.chat import (
13 + ChatCompletionMessageParam as OpenAIChatCompletionMessageParam)
14 +# yapf: enable
15 +# pydantic needs the TypedDict from typing_extensions
16 +from pydantic import ConfigDict
17 from transformers import PreTrainedTokenizer
18 +from typing_extensions import Required, TypedDict
19
20 from vllm.config import ModelConfig
11 -from vllm.entrypoints.openai.protocol import (ChatCompletionContentPartParam,
12 - ChatCompletionMessageParam)
21 from vllm.logger import init_logger
22 from vllm.multimodal import MultiModalDataDict
23 from vllm.multimodal.utils import async_get_and_parse_image
@@ -17,6 +25,37 @@ from vllm.multimodal.utils import async_get_and_parse_image
25 logger = init_logger(__name__)
26
27
28 +class CustomChatCompletionContentPartParam(TypedDict, total=False):
29 + __pydantic_config__ = ConfigDict(extra="allow") # type: ignore
30 +
31 + type: Required[str]
32 + """The type of the content part."""
33 +
34 +
35 +ChatCompletionContentPartParam = Union[OpenAIChatCompletionContentPartParam,
36 + CustomChatCompletionContentPartParam]
37 +
38 +
39 +class CustomChatCompletionMessageParam(TypedDict, total=False):
40 + """Enables custom roles in the Chat Completion API."""
41 + role: Required[str]
42 + """The role of the message's author."""
43 +
44 + content: Union[str, List[ChatCompletionContentPartParam]]
45 + """The contents of the message."""
46 +
47 + name: str
48 + """An optional name for the participant.
49 +
50 + Provides the model information to differentiate between participants of the
51 + same role.
52 + """
53 +
54 +
55 +ChatCompletionMessageParam = Union[OpenAIChatCompletionMessageParam,
56 + CustomChatCompletionMessageParam]
57 +
58 +
59 @final # So that it should be compatible with Dict[str, str]
60 class ConversationMessage(TypedDict):
61 role: str
vllm/entrypoints/openai/protocol.py
+2 -36
@@ -3,50 +3,16 @@
3 import time
4 from typing import Any, Dict, List, Literal, Optional, Union
5
6 -import openai.types.chat
6 import torch
7 from pydantic import BaseModel, ConfigDict, Field, model_validator
9 -# pydantic needs the TypedDict from typing_extensions
10 -from typing_extensions import Annotated, Required, TypedDict
8 +from typing_extensions import Annotated
9
10 +from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
11 from vllm.pooling_params import PoolingParams
12 from vllm.sampling_params import SamplingParams
13 from vllm.utils import random_uuid
14
15
17 -class CustomChatCompletionContentPartParam(TypedDict, total=False):
18 - __pydantic_config__ = ConfigDict(extra="allow") # type: ignore
19 -
20 - type: Required[str]
21 - """The type of the content part."""
22 -
23 -
24 -ChatCompletionContentPartParam = Union[
25 - openai.types.chat.ChatCompletionContentPartParam,
26 - CustomChatCompletionContentPartParam]
27 -
28 -
29 -class CustomChatCompletionMessageParam(TypedDict, total=False):
30 - """Enables custom roles in the Chat Completion API."""
31 - role: Required[str]
32 - """The role of the message's author."""
33 -
34 - content: Union[str, List[ChatCompletionContentPartParam]]
35 - """The contents of the message."""
36 -
37 - name: str
38 - """An optional name for the participant.
39 -
40 - Provides the model information to differentiate between participants of the
41 - same role.
42 - """
43 -
44 -
45 -ChatCompletionMessageParam = Union[
46 - openai.types.chat.ChatCompletionMessageParam,
47 - CustomChatCompletionMessageParam]
48 -
49 -
16 class OpenAIBaseModel(BaseModel):
17 # OpenAI API does not allow extra fields
18 model_config = ConfigDict(extra="forbid")
vllm/entrypoints/openai/serving_chat.py
+3 -3
@@ -9,9 +9,9 @@ from transformers import PreTrainedTokenizer
9
10 from vllm.config import ModelConfig
11 from vllm.engine.async_llm_engine import AsyncLLMEngine
12 -from vllm.entrypoints.openai.chat_utils import (ConversationMessage,
13 - load_chat_template,
14 - parse_chat_message_content)
12 +from vllm.entrypoints.chat_utils import (ConversationMessage,
13 + load_chat_template,
14 + parse_chat_message_content)
15 from vllm.entrypoints.openai.protocol import (
16 ChatCompletionLogProb, ChatCompletionLogProbs,
17 ChatCompletionLogProbsContent, ChatCompletionNamedToolChoiceParam,
vllm/entrypoints/openai/serving_tokenization.py
+3 -3
@@ -2,9 +2,9 @@ from typing import List, Optional
2
3 from vllm.config import ModelConfig
4 from vllm.engine.async_llm_engine import AsyncLLMEngine
5 -from vllm.entrypoints.openai.chat_utils import (ConversationMessage,
6 - load_chat_template,
7 - parse_chat_message_content)
5 +from vllm.entrypoints.chat_utils import (ConversationMessage,
6 + load_chat_template,
7 + parse_chat_message_content)
8 from vllm.entrypoints.openai.protocol import (DetokenizeRequest,
9 DetokenizeResponse,
10 TokenizeRequest,