A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Frontend] Support for chat completions input in the tokenize endpoint (#5923)
sasha0552 committed
Jul 16, 2024 at 12:18 UTC
7a3d2a5b957063e911c25401c593dbc7798a5536
9 files changed
+386
-244
tests/async_engine/test_chat_template.py
+5
-9
@@ -4,8 +4,8 @@ from dataclasses import dataclass
4
5
import pytest
6
7
+from vllm.entrypoints.openai.chat_utils import load_chat_template
8
from vllm.entrypoints.openai.protocol import ChatCompletionRequest
8
-from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
9
from vllm.transformers_utils.tokenizer import get_tokenizer
10
11
chatml_jinja_path = pathlib.Path(os.path.dirname(os.path.abspath(
@@ -64,8 +64,7 @@ def test_load_chat_template():
64
# Testing chatml template
65
tokenizer = MockTokenizer()
66
mock_serving_chat = MockServingChat(tokenizer)
67
- OpenAIServingChat._load_chat_template(mock_serving_chat,
68
- chat_template=chatml_jinja_path)
67
+ load_chat_template(mock_serving_chat, chat_template=chatml_jinja_path)
68
69
template_content = tokenizer.chat_template
70
@@ -84,8 +83,7 @@ def test_no_load_chat_template_filelike():
83
mock_serving_chat = MockServingChat(tokenizer)
84
85
with pytest.raises(ValueError, match="looks like a file path"):
87
- OpenAIServingChat._load_chat_template(mock_serving_chat,
88
- chat_template=template)
86
+ load_chat_template(mock_serving_chat, chat_template=template)
87
88
89
def test_no_load_chat_template_literallike():
@@ -94,8 +92,7 @@ def test_no_load_chat_template_literallike():
92
tokenizer = MockTokenizer()
93
94
mock_serving_chat = MockServingChat(tokenizer)
97
- OpenAIServingChat._load_chat_template(mock_serving_chat,
98
- chat_template=template)
95
+ load_chat_template(mock_serving_chat, chat_template=template)
96
template_content = tokenizer.chat_template
97
98
assert template_content == template
@@ -109,8 +106,7 @@ def test_get_gen_prompt(model, template, add_generation_prompt,
106
# Initialize the tokenizer
107
tokenizer = get_tokenizer(tokenizer_name=model)
108
mock_serving_chat = MockServingChat(tokenizer)
112
- OpenAIServingChat._load_chat_template(mock_serving_chat,
113
- chat_template=template)
109
+ load_chat_template(mock_serving_chat, chat_template=template)
110
111
# Create a mock request object using keyword arguments
112
mock_request = ChatCompletionRequest(
tests/entrypoints/openai/test_completion.py
-49
@@ -6,7 +6,6 @@ from typing import List
6
import jsonschema
7
import openai # use the official client for correctness check
8
import pytest
9
-import requests
9
# downloading lora to test lora requests
10
from huggingface_hub import snapshot_download
11
from openai import BadRequestError
@@ -636,51 +635,3 @@ async def test_guided_decoding_type_error(client: openai.AsyncOpenAI,
635
prompt="Give an example string that fits this regex",
636
extra_body=dict(guided_regex=sample_regex,
637
guided_json=sample_json_schema))
639
-
640
-
641
-@pytest.mark.asyncio
642
-@pytest.mark.parametrize(
643
- "model_name",
644
- [MODEL_NAME],
645
-)
646
-async def test_tokenize(client: openai.AsyncOpenAI, model_name: str):
647
- base_url = str(client.base_url)[:-3].strip("/")
648
- tokenizer = get_tokenizer(tokenizer_name=model_name, tokenizer_mode="fast")
649
-
650
- for add_special in [False, True]:
651
- prompt = "This is a test prompt."
652
- tokens = tokenizer.encode(prompt, add_special_tokens=add_special)
653
-
654
- response = requests.post(base_url + "/tokenize",
655
- json={
656
- "add_special_tokens": add_special,
657
- "model": model_name,
658
- "prompt": prompt
659
- })
660
- response.raise_for_status()
661
- assert response.json() == {
662
- "tokens": tokens,
663
- "count": len(tokens),
664
- "max_model_len": 8192
665
- }
666
-
667
-
668
-@pytest.mark.asyncio
669
-@pytest.mark.parametrize(
670
- "model_name",
671
- [MODEL_NAME],
672
-)
673
-async def test_detokenize(client: openai.AsyncOpenAI, model_name: str):
674
- base_url = str(client.base_url)[:-3]
675
- tokenizer = get_tokenizer(tokenizer_name=model_name, tokenizer_mode="fast")
676
-
677
- prompt = "This is a test prompt."
678
- tokens = tokenizer.encode(prompt, add_special_tokens=False)
679
-
680
- response = requests.post(base_url + "detokenize",
681
- json={
682
- "model": model_name,
683
- "tokens": tokens
684
- })
685
- response.raise_for_status()
686
- assert response.json() == {"prompt": prompt}
tests/entrypoints/openai/test_tokenization.py
new
+128
@@ -0,0 +1,128 @@
1
+import openai # use the official client for correctness check
2
+import pytest
3
+import requests
4
+
5
+from vllm.transformers_utils.tokenizer import get_tokenizer
6
+
7
+from ...utils import RemoteOpenAIServer
8
+
9
+# any model with a chat template should work here
10
+MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
11
+
12
+
13
+@pytest.fixture(scope="module")
14
+def server():
15
+ with RemoteOpenAIServer([
16
+ "--model",
17
+ MODEL_NAME,
18
+ # use half precision for speed and memory savings in CI environment
19
+ "--dtype",
20
+ "bfloat16",
21
+ "--max-model-len",
22
+ "8192",
23
+ "--enforce-eager",
24
+ "--max-num-seqs",
25
+ "128",
26
+ ]) as remote_server:
27
+ yield remote_server
28
+
29
+
30
+@pytest.fixture(scope="module")
31
+def client(server):
32
+ return server.get_async_client()
33
+
34
+
35
+@pytest.mark.asyncio
36
+@pytest.mark.parametrize(
37
+ "model_name",
38
+ [MODEL_NAME],
39
+)
40
+async def test_tokenize_completions(client: openai.AsyncOpenAI,
41
+ model_name: str):
42
+ base_url = str(client.base_url)[:-3].strip("/")
43
+ tokenizer = get_tokenizer(tokenizer_name=model_name, tokenizer_mode="fast")
44
+
45
+ for add_special in [False, True]:
46
+ prompt = "This is a test prompt."
47
+ tokens = tokenizer.encode(prompt, add_special_tokens=add_special)
48
+
49
+ response = requests.post(base_url + "/tokenize",
50
+ json={
51
+ "add_special_tokens": add_special,
52
+ "model": model_name,
53
+ "prompt": prompt
54
+ })
55
+ response.raise_for_status()
56
+
57
+ assert response.json() == {
58
+ "tokens": tokens,
59
+ "count": len(tokens),
60
+ "max_model_len": 8192
61
+ }
62
+
63
+
64
+@pytest.mark.asyncio
65
+@pytest.mark.parametrize(
66
+ "model_name",
67
+ [MODEL_NAME],
68
+)
69
+async def test_tokenize_chat(client: openai.AsyncOpenAI, model_name: str):
70
+ base_url = str(client.base_url)[:-3].strip("/")
71
+ tokenizer = get_tokenizer(tokenizer_name=model_name, tokenizer_mode="fast")
72
+
73
+ for add_generation in [False, True]:
74
+ for add_special in [False, True]:
75
+ conversation = [{
76
+ "role": "user",
77
+ "content": "Hi there!"
78
+ }, {
79
+ "role": "assistant",
80
+ "content": "Nice to meet you!"
81
+ }, {
82
+ "role": "user",
83
+ "content": "Can I ask a question?"
84
+ }]
85
+
86
+ prompt = tokenizer.apply_chat_template(
87
+ add_generation_prompt=add_generation,
88
+ conversation=conversation,
89
+ tokenize=False)
90
+ tokens = tokenizer.encode(prompt, add_special_tokens=add_special)
91
+
92
+ response = requests.post(base_url + "/tokenize",
93
+ json={
94
+ "add_generation_prompt":
95
+ add_generation,
96
+ "add_special_tokens": add_special,
97
+ "messages": conversation,
98
+ "model": model_name
99
+ })
100
+ response.raise_for_status()
101
+
102
+ assert response.json() == {
103
+ "tokens": tokens,
104
+ "count": len(tokens),
105
+ "max_model_len": 8192
106
+ }
107
+
108
+
109
+@pytest.mark.asyncio
110
+@pytest.mark.parametrize(
111
+ "model_name",
112
+ [MODEL_NAME],
113
+)
114
+async def test_detokenize(client: openai.AsyncOpenAI, model_name: str):
115
+ base_url = str(client.base_url)[:-3].strip("/")
116
+ tokenizer = get_tokenizer(tokenizer_name=model_name, tokenizer_mode="fast")
117
+
118
+ prompt = "This is a test prompt."
119
+ tokens = tokenizer.encode(prompt, add_special_tokens=False)
120
+
121
+ response = requests.post(base_url + "/detokenize",
122
+ json={
123
+ "model": model_name,
124
+ "tokens": tokens
125
+ })
126
+ response.raise_for_status()
127
+
128
+ assert response.json() == {"prompt": prompt}
vllm/entrypoints/openai/api_server.py
+8
-2
@@ -33,6 +33,8 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
33
from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
34
from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
35
from vllm.entrypoints.openai.serving_embedding import OpenAIServingEmbedding
36
+from vllm.entrypoints.openai.serving_tokenization import (
37
+ OpenAIServingTokenization)
38
from vllm.logger import init_logger
39
from vllm.usage.usage_lib import UsageContext
40
from vllm.utils import FlexibleArgumentParser
@@ -46,6 +48,7 @@ engine_args: AsyncEngineArgs
48
openai_serving_chat: OpenAIServingChat
49
openai_serving_completion: OpenAIServingCompletion
50
openai_serving_embedding: OpenAIServingEmbedding
51
+openai_serving_tokenization: OpenAIServingTokenization
52
53
logger = init_logger('vllm.entrypoints.openai.api_server')
54
@@ -86,7 +89,7 @@ async def health() -> Response:
89
90
@router.post("/tokenize")
91
async def tokenize(request: TokenizeRequest):
89
- generator = await openai_serving_completion.create_tokenize(request)
92
+ generator = await openai_serving_tokenization.create_tokenize(request)
93
if isinstance(generator, ErrorResponse):
94
return JSONResponse(content=generator.model_dump(),
95
status_code=generator.code)
@@ -97,7 +100,7 @@ async def tokenize(request: TokenizeRequest):
100
101
@router.post("/detokenize")
102
async def detokenize(request: DetokenizeRequest):
100
- generator = await openai_serving_completion.create_detokenize(request)
103
+ generator = await openai_serving_tokenization.create_detokenize(request)
104
if isinstance(generator, ErrorResponse):
105
return JSONResponse(content=generator.model_dump(),
106
status_code=generator.code)
@@ -241,6 +244,7 @@ def run_server(args, llm_engine=None):
244
global openai_serving_chat
245
global openai_serving_completion
246
global openai_serving_embedding
247
+ global openai_serving_tokenization
248
249
openai_serving_chat = OpenAIServingChat(engine, model_config,
250
served_model_names,
@@ -252,6 +256,8 @@ def run_server(args, llm_engine=None):
256
args.prompt_adapters)
257
openai_serving_embedding = OpenAIServingEmbedding(engine, model_config,
258
served_model_names)
259
+ openai_serving_tokenization = OpenAIServingTokenization(
260
+ engine, model_config, served_model_names, args.chat_template)
261
app.root_path = args.root_path
262
263
logger.info("Available routes are:")
vllm/entrypoints/openai/chat_utils.py
new
+156
@@ -0,0 +1,156 @@
1
+import codecs
2
+from dataclasses import dataclass, field
3
+from functools import lru_cache
4
+from typing import Awaitable, Iterable, List, Optional, TypedDict, cast, final
5
+
6
+from openai.types.chat import (ChatCompletionContentPartImageParam,
7
+ ChatCompletionContentPartTextParam)
8
+
9
+from vllm.entrypoints.openai.protocol import (ChatCompletionContentPartParam,
10
+ ChatCompletionMessageParam)
11
+from vllm.entrypoints.openai.serving_engine import OpenAIServing
12
+from vllm.logger import init_logger
13
+from vllm.multimodal import MultiModalDataDict
14
+from vllm.multimodal.utils import async_get_and_parse_image
15
+
16
+logger = init_logger(__name__)
17
+
18
+
19
+@final # So that it should be compatible with Dict[str, str]
20
+class ConversationMessage(TypedDict):
21
+ role: str
22
+ content: str
23
+
24
+
25
+@dataclass(frozen=True)
26
+class ChatMessageParseResult:
27
+ messages: List[ConversationMessage]
28
+ mm_futures: List[Awaitable[MultiModalDataDict]] = field(
29
+ default_factory=list)
30
+
31
+
32
+def load_chat_template(engine: OpenAIServing, chat_template: Optional[str]):
33
+ tokenizer = engine.tokenizer
34
+
35
+ if chat_template is not None:
36
+ try:
37
+ with open(chat_template, "r") as f:
38
+ tokenizer.chat_template = f.read()
39
+ except OSError as e:
40
+ JINJA_CHARS = "{}\n"
41
+ if not any(c in chat_template for c in JINJA_CHARS):
42
+ msg = (f"The supplied chat template ({chat_template}) "
43
+ f"looks like a file path, but it failed to be "
44
+ f"opened. Reason: {e}")
45
+ raise ValueError(msg) from e
46
+
47
+ # If opening a file fails, set chat template to be args to
48
+ # ensure we decode so our escape are interpreted correctly
49
+ tokenizer.chat_template = codecs.decode(chat_template,
50
+ "unicode_escape")
51
+
52
+ logger.info("Using supplied chat template:\n%s",
53
+ tokenizer.chat_template)
54
+ elif tokenizer.chat_template is not None:
55
+ logger.info("Using default chat template:\n%s",
56
+ tokenizer.chat_template)
57
+ else:
58
+ logger.warning("No chat template provided. Chat API will not work.")
59
+
60
+
61
+@lru_cache(maxsize=None)
62
+def _image_token_str(engine: OpenAIServing) -> Optional[str]:
63
+ # TODO: Let user specify how to insert image tokens into prompt
64
+ # (similar to chat template)
65
+ model_type = engine.model_config.hf_config.model_type
66
+ if model_type == "phi3_v":
67
+ # Workaround since this token is not defined in the tokenizer
68
+ return "<|image_1|>"
69
+ if model_type in ("blip-2", "chatglm", "fuyu", "minicpmv", "paligemma"):
70
+ # These models do not use image tokens in the prompt
71
+ return None
72
+ if model_type.startswith("llava"):
73
+ return engine.tokenizer.decode(
74
+ engine.model_config.hf_config.image_token_index)
75
+
76
+ else:
77
+ raise TypeError("Unknown model type: {model_type}")
78
+
79
+
80
+# TODO: Let user specify how to insert image tokens into prompt
81
+# (similar to chat template)
82
+def _get_full_image_text_prompt(engine: OpenAIServing, image_token_str: str,
83
+ text_prompt: str) -> str:
84
+ """Combine image and text prompts for vision language model"""
85
+
86
+ # NOTE: For now we assume all model architectures use the same
87
+ # image + text prompt format. This may change in the future.
88
+ return f"{image_token_str}\n{text_prompt}"
89
+
90
+
91
+def _parse_chat_message_content_parts(
92
+ engine: OpenAIServing,
93
+ role: str,
94
+ parts: Iterable[ChatCompletionContentPartParam],
95
+) -> ChatMessageParseResult:
96
+ texts: List[str] = []
97
+ mm_futures: List[Awaitable[MultiModalDataDict]] = []
98
+
99
+ for part in parts:
100
+ part_type = part["type"]
101
+ if part_type == "text":
102
+ text = cast(ChatCompletionContentPartTextParam, part)["text"]
103
+ texts.append(text)
104
+ elif part_type == "image_url":
105
+ if len(mm_futures) > 0:
106
+ raise NotImplementedError(
107
+ "Multiple 'image_url' input is currently not supported.")
108
+
109
+ image_url = cast(ChatCompletionContentPartImageParam,
110
+ part)["image_url"]
111
+
112
+ if image_url.get("detail", "auto") != "auto":
113
+ logger.warning(
114
+ "'image_url.detail' is currently not supported and "
115
+ "will be ignored.")
116
+
117
+ image_future = async_get_and_parse_image(image_url["url"])
118
+ mm_futures.append(image_future)
119
+ else:
120
+ raise NotImplementedError(f"Unknown part type: {part_type}")
121
+
122
+ text_prompt = "\n".join(texts)
123
+
124
+ if mm_futures:
125
+ image_token_str = _image_token_str(engine)
126
+ if image_token_str is not None:
127
+ if image_token_str in text_prompt:
128
+ logger.warning(
129
+ "Detected image token string in the text prompt. "
130
+ "Skipping prompt formatting.")
131
+ else:
132
+ text_prompt = _get_full_image_text_prompt(
133
+ engine,
134
+ image_token_str=image_token_str,
135
+ text_prompt=text_prompt,
136
+ )
137
+
138
+ messages = [ConversationMessage(role=role, content=text_prompt)]
139
+
140
+ return ChatMessageParseResult(messages=messages, mm_futures=mm_futures)
141
+
142
+
143
+def parse_chat_message_content(
144
+ engine: OpenAIServing,
145
+ message: ChatCompletionMessageParam,
146
+) -> ChatMessageParseResult:
147
+ role = message["role"]
148
+ content = message.get("content")
149
+
150
+ if content is None:
151
+ return ChatMessageParseResult(messages=[], mm_futures=[])
152
+ if isinstance(content, str):
153
+ messages = [ConversationMessage(role=role, content=content)]
154
+ return ChatMessageParseResult(messages=messages, mm_futures=[])
155
+
156
+ return _parse_chat_message_content_parts(engine, role, content)
vllm/entrypoints/openai/protocol.py
+5
-3
@@ -738,15 +738,17 @@ class BatchRequestOutput(OpenAIBaseModel):
738
739
740
class TokenizeRequest(OpenAIBaseModel):
741
+ add_generation_prompt: bool = Field(default=True)
742
+ add_special_tokens: bool = Field(default=False)
743
+ prompt: Optional[str] = Field(default=None)
744
+ messages: Optional[List[ChatCompletionMessageParam]] = Field(default=None)
745
model: str
742
- prompt: str
743
- add_special_tokens: bool = Field(default=True)
746
747
748
class TokenizeResponse(OpenAIBaseModel):
747
- tokens: List[int]
749
count: int
750
max_model_len: int
751
+ tokens: List[int]
752
753
754
class DetokenizeRequest(OpenAIBaseModel):
vllm/entrypoints/openai/serving_chat.py
+10
-151
@@ -1,22 +1,19 @@
1
-import codecs
1
import time
3
-from dataclasses import dataclass, field
4
-from functools import cached_property
5
-from typing import (AsyncGenerator, AsyncIterator, Awaitable, Dict, Iterable,
6
- List, Optional)
2
+from typing import (AsyncGenerator, AsyncIterator, Awaitable, Dict, List,
3
+ Optional)
4
from typing import Sequence as GenericSequence
8
-from typing import TypedDict, Union, cast, final
5
+from typing import Union
6
7
from fastapi import Request
11
-from openai.types.chat import (ChatCompletionContentPartImageParam,
12
- ChatCompletionContentPartTextParam)
8
9
from vllm.config import ModelConfig
10
from vllm.engine.async_llm_engine import AsyncLLMEngine
11
+from vllm.entrypoints.openai.chat_utils import (ConversationMessage,
12
+ load_chat_template,
13
+ parse_chat_message_content)
14
from vllm.entrypoints.openai.protocol import (
17
- ChatCompletionContentPartParam, ChatCompletionLogProb,
18
- ChatCompletionLogProbs, ChatCompletionLogProbsContent,
19
- ChatCompletionMessageParam, ChatCompletionNamedToolChoiceParam,
15
+ ChatCompletionLogProb, ChatCompletionLogProbs,
16
+ ChatCompletionLogProbsContent, ChatCompletionNamedToolChoiceParam,
17
ChatCompletionRequest, ChatCompletionResponse,
18
ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice,
19
ChatCompletionStreamResponse, ChatMessage, DeltaMessage, ErrorResponse,
@@ -28,7 +25,6 @@ from vllm.logger import init_logger
25
from vllm.model_executor.guided_decoding import (
26
get_guided_decoding_logits_processor)
27
from vllm.multimodal import MultiModalDataDict
31
-from vllm.multimodal.utils import async_get_and_parse_image
28
from vllm.outputs import RequestOutput
29
from vllm.sequence import Logprob
30
from vllm.tracing import (contains_trace_headers, extract_trace_headers,
@@ -38,19 +34,6 @@ from vllm.utils import random_uuid
34
logger = init_logger(__name__)
35
36
41
-@final # So that it should be compatible with Dict[str, str]
42
-class ConversationMessage(TypedDict):
43
- role: str
44
- content: str
45
-
46
-
47
-@dataclass(frozen=True)
48
-class ChatMessageParseResult:
49
- messages: List[ConversationMessage]
50
- mm_futures: List[Awaitable[MultiModalDataDict]] = field(
51
- default_factory=list)
52
-
53
-
37
class OpenAIServingChat(OpenAIServing):
38
39
def __init__(self,
@@ -66,131 +49,7 @@ class OpenAIServingChat(OpenAIServing):
49
lora_modules=lora_modules)
50
51
self.response_role = response_role
69
- self._load_chat_template(chat_template)
70
-
71
- def _load_chat_template(self, chat_template: Optional[str]):
72
- tokenizer = self.tokenizer
73
-
74
- if chat_template is not None:
75
- try:
76
- with open(chat_template, "r") as f:
77
- tokenizer.chat_template = f.read()
78
- except OSError as e:
79
- JINJA_CHARS = "{}\n"
80
- if not any(c in chat_template for c in JINJA_CHARS):
81
- msg = (f"The supplied chat template ({chat_template}) "
82
- f"looks like a file path, but it failed to be "
83
- f"opened. Reason: {e}")
84
- raise ValueError(msg) from e
85
-
86
- # If opening a file fails, set chat template to be args to
87
- # ensure we decode so our escape are interpreted correctly
88
- tokenizer.chat_template = codecs.decode(
89
- chat_template, "unicode_escape")
90
-
91
- logger.info("Using supplied chat template:\n%s",
92
- tokenizer.chat_template)
93
- elif tokenizer.chat_template is not None:
94
- logger.info("Using default chat template:\n%s",
95
- tokenizer.chat_template)
96
- else:
97
- logger.warning(
98
- "No chat template provided. Chat API will not work.")
99
-
100
- @cached_property
101
- def image_token_str(self) -> Optional[str]:
102
- # TODO: Let user specify how to insert image tokens into prompt
103
- # (similar to chat template)
104
- model_type = self.model_config.hf_config.model_type
105
- if model_type == "phi3_v":
106
- # Workaround since this token is not defined in the tokenizer
107
- return "<|image_1|>"
108
- if model_type in ("blip-2", "chatglm", "fuyu", "minicpmv",
109
- "paligemma"):
110
- # These models do not use image tokens in the prompt
111
- return None
112
- if model_type.startswith("llava"):
113
- return self.tokenizer.decode(
114
- self.model_config.hf_config.image_token_index)
115
-
116
- else:
117
- raise TypeError("Unknown model type: {model_type}")
118
-
119
- # TODO: Let user specify how to insert image tokens into prompt
120
- # (similar to chat template)
121
- def _get_full_image_text_prompt(self, image_token_str: str,
122
- text_prompt: str) -> str:
123
- """Combine image and text prompts for vision language model"""
124
-
125
- # NOTE: For now we assume all model architectures use the same
126
- # image + text prompt format. This may change in the future.
127
- return f"{image_token_str}\n{text_prompt}"
128
-
129
- def _parse_chat_message_content_parts(
130
- self,
131
- role: str,
132
- parts: Iterable[ChatCompletionContentPartParam],
133
- ) -> ChatMessageParseResult:
134
- texts: List[str] = []
135
- mm_futures: List[Awaitable[MultiModalDataDict]] = []
136
-
137
- for part in parts:
138
- part_type = part["type"]
139
- if part_type == "text":
140
- text = cast(ChatCompletionContentPartTextParam, part)["text"]
141
- texts.append(text)
142
- elif part_type == "image_url":
143
- if len(mm_futures) > 0:
144
- raise NotImplementedError(
145
- "Multiple 'image_url' input is currently not supported."
146
- )
147
-
148
- image_url = cast(ChatCompletionContentPartImageParam,
149
- part)["image_url"]
150
-
151
- if image_url.get("detail", "auto") != "auto":
152
- logger.warning(
153
- "'image_url.detail' is currently not supported and "
154
- "will be ignored.")
155
-
156
- image_future = async_get_and_parse_image(image_url["url"])
157
- mm_futures.append(image_future)
158
- else:
159
- raise NotImplementedError(f"Unknown part type: {part_type}")
160
-
161
- text_prompt = "\n".join(texts)
162
-
163
- if mm_futures:
164
- image_token_str = self.image_token_str
165
- if image_token_str is not None:
166
- if image_token_str in text_prompt:
167
- logger.warning(
168
- "Detected image token string in the text prompt. "
169
- "Skipping prompt formatting.")
170
- else:
171
- text_prompt = self._get_full_image_text_prompt(
172
- image_token_str=image_token_str,
173
- text_prompt=text_prompt,
174
- )
175
-
176
- messages = [ConversationMessage(role=role, content=text_prompt)]
177
-
178
- return ChatMessageParseResult(messages=messages, mm_futures=mm_futures)
179
-
180
- def _parse_chat_message_content(
181
- self,
182
- message: ChatCompletionMessageParam,
183
- ) -> ChatMessageParseResult:
184
- role = message["role"]
185
- content = message.get("content")
186
-
187
- if content is None:
188
- return ChatMessageParseResult(messages=[], mm_futures=[])
189
- if isinstance(content, str):
190
- messages = [ConversationMessage(role=role, content=content)]
191
- return ChatMessageParseResult(messages=messages, mm_futures=[])
192
-
193
- return self._parse_chat_message_content_parts(role, content)
52
+ load_chat_template(self, chat_template)
53
54
async def create_chat_completion(
55
self,
@@ -216,7 +75,7 @@ class OpenAIServingChat(OpenAIServing):
75
mm_futures: List[Awaitable[MultiModalDataDict]] = []
76
77
for msg in request.messages:
219
- chat_parsed_result = self._parse_chat_message_content(msg)
78
+ chat_parsed_result = parse_chat_message_content(self, msg)
79
80
conversation.extend(chat_parsed_result.messages)
81
mm_futures.extend(chat_parsed_result.mm_futures)
vllm/entrypoints/openai/serving_completion.py
+1
-30
@@ -16,10 +16,7 @@ from vllm.entrypoints.openai.protocol import (CompletionLogProbs,
16
CompletionResponseChoice,
17
CompletionResponseStreamChoice,
18
CompletionStreamResponse,
19
- DetokenizeRequest,
20
- DetokenizeResponse,
21
- TokenizeRequest,
22
- TokenizeResponse, UsageInfo)
19
+ UsageInfo)
20
# yapf: enable
21
from vllm.entrypoints.openai.serving_engine import (LoRAModulePath,
22
OpenAIServing,
@@ -457,29 +454,3 @@ class OpenAIServingCompletion(OpenAIServing):
454
tokens=out_tokens,
455
top_logprobs=out_top_logprobs,
456
)
460
-
461
- async def create_tokenize(self,
462
- request: TokenizeRequest) -> TokenizeResponse:
463
- error_check_ret = await self._check_model(request)
464
- if error_check_ret is not None:
465
- return error_check_ret
466
-
467
- (input_ids, input_text) = self._validate_prompt_and_tokenize(
468
- request,
469
- prompt=request.prompt,
470
- add_special_tokens=request.add_special_tokens)
471
-
472
- return TokenizeResponse(tokens=input_ids,
473
- count=len(input_ids),
474
- max_model_len=self.max_model_len)
475
-
476
- async def create_detokenize(
477
- self, request: DetokenizeRequest) -> DetokenizeResponse:
478
- error_check_ret = await self._check_model(request)
479
- if error_check_ret is not None:
480
- return error_check_ret
481
-
482
- (input_ids, input_text) = self._validate_prompt_and_tokenize(
483
- request, prompt_ids=request.tokens)
484
-
485
- return DetokenizeResponse(prompt=input_text)
vllm/entrypoints/openai/serving_tokenization.py
new
+73
@@ -0,0 +1,73 @@
1
+from typing import List, Optional
2
+
3
+from vllm.config import ModelConfig
4
+from vllm.engine.async_llm_engine import AsyncLLMEngine
5
+from vllm.entrypoints.openai.chat_utils import (ConversationMessage,
6
+ load_chat_template,
7
+ parse_chat_message_content)
8
+from vllm.entrypoints.openai.protocol import (DetokenizeRequest,
9
+ DetokenizeResponse,
10
+ TokenizeRequest,
11
+ TokenizeResponse)
12
+from vllm.entrypoints.openai.serving_engine import OpenAIServing
13
+
14
+
15
+class OpenAIServingTokenization(OpenAIServing):
16
+
17
+ def __init__(self,
18
+ engine: AsyncLLMEngine,
19
+ model_config: ModelConfig,
20
+ served_model_names: List[str],
21
+ chat_template: Optional[str] = None):
22
+ super().__init__(engine=engine,
23
+ model_config=model_config,
24
+ served_model_names=served_model_names,
25
+ lora_modules=None)
26
+
27
+ load_chat_template(self, chat_template)
28
+
29
+ async def create_tokenize(self,
30
+ request: TokenizeRequest) -> TokenizeResponse:
31
+ error_check_ret = await self._check_model(request)
32
+ if error_check_ret is not None:
33
+ return error_check_ret
34
+
35
+ if not (request.prompt or request.messages):
36
+ return self.create_error_response(
37
+ "Either `prompt` or `messages` should be provided.")
38
+
39
+ if (request.prompt and request.messages):
40
+ return self.create_error_response(
41
+ "Only one of `prompt` or `messages` should be provided.")
42
+
43
+ if request.messages:
44
+ conversation: List[ConversationMessage] = []
45
+
46
+ for message in request.messages:
47
+ conversation.extend(
48
+ parse_chat_message_content(self, message).messages)
49
+
50
+ request.prompt = self.tokenizer.apply_chat_template(
51
+ add_generation_prompt=request.add_generation_prompt,
52
+ conversation=conversation,
53
+ tokenize=False)
54
+
55
+ (input_ids, input_text) = self._validate_prompt_and_tokenize(
56
+ request,
57
+ prompt=request.prompt,
58
+ add_special_tokens=request.add_special_tokens)
59
+
60
+ return TokenizeResponse(tokens=input_ids,
61
+ count=len(input_ids),
62
+ max_model_len=self.max_model_len)
63
+
64
+ async def create_detokenize(
65
+ self, request: DetokenizeRequest) -> DetokenizeResponse:
66
+ error_check_ret = await self._check_model(request)
67
+ if error_check_ret is not None:
68
+ return error_check_ret
69
+
70
+ (input_ids, input_text) = self._validate_prompt_and_tokenize(
71
+ request, prompt_ids=request.tokens)
72
+
73
+ return DetokenizeResponse(prompt=input_text)