@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7a3d2a5b95
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Frontend] Support for chat completions input in the tokenize endpoint (#5923)

sasha0552 committed Jul 16, 2024 at 12:18 UTC 7a3d2a5b957063e911c25401c593dbc7798a5536
9 files changed +386 -244
tests/async_engine/test_chat_template.py
+5 -9
@@ -4,8 +4,8 @@ from dataclasses import dataclass
4
5 import pytest
6
7 +from vllm.entrypoints.openai.chat_utils import load_chat_template
8 from vllm.entrypoints.openai.protocol import ChatCompletionRequest
8 -from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
9 from vllm.transformers_utils.tokenizer import get_tokenizer
10
11 chatml_jinja_path = pathlib.Path(os.path.dirname(os.path.abspath(
@@ -64,8 +64,7 @@ def test_load_chat_template():
64 # Testing chatml template
65 tokenizer = MockTokenizer()
66 mock_serving_chat = MockServingChat(tokenizer)
67 - OpenAIServingChat._load_chat_template(mock_serving_chat,
68 - chat_template=chatml_jinja_path)
67 + load_chat_template(mock_serving_chat, chat_template=chatml_jinja_path)
68
69 template_content = tokenizer.chat_template
70
@@ -84,8 +83,7 @@ def test_no_load_chat_template_filelike():
83 mock_serving_chat = MockServingChat(tokenizer)
84
85 with pytest.raises(ValueError, match="looks like a file path"):
87 - OpenAIServingChat._load_chat_template(mock_serving_chat,
88 - chat_template=template)
86 + load_chat_template(mock_serving_chat, chat_template=template)
87
88
89 def test_no_load_chat_template_literallike():
@@ -94,8 +92,7 @@ def test_no_load_chat_template_literallike():
92 tokenizer = MockTokenizer()
93
94 mock_serving_chat = MockServingChat(tokenizer)
97 - OpenAIServingChat._load_chat_template(mock_serving_chat,
98 - chat_template=template)
95 + load_chat_template(mock_serving_chat, chat_template=template)
96 template_content = tokenizer.chat_template
97
98 assert template_content == template
@@ -109,8 +106,7 @@ def test_get_gen_prompt(model, template, add_generation_prompt,
106 # Initialize the tokenizer
107 tokenizer = get_tokenizer(tokenizer_name=model)
108 mock_serving_chat = MockServingChat(tokenizer)
112 - OpenAIServingChat._load_chat_template(mock_serving_chat,
113 - chat_template=template)
109 + load_chat_template(mock_serving_chat, chat_template=template)
110
111 # Create a mock request object using keyword arguments
112 mock_request = ChatCompletionRequest(
tests/entrypoints/openai/test_completion.py
-49
@@ -6,7 +6,6 @@ from typing import List
6 import jsonschema
7 import openai # use the official client for correctness check
8 import pytest
9 -import requests
9 # downloading lora to test lora requests
10 from huggingface_hub import snapshot_download
11 from openai import BadRequestError
@@ -636,51 +635,3 @@ async def test_guided_decoding_type_error(client: openai.AsyncOpenAI,
635 prompt="Give an example string that fits this regex",
636 extra_body=dict(guided_regex=sample_regex,
637 guided_json=sample_json_schema))
639 -
640 -
641 -@pytest.mark.asyncio
642 -@pytest.mark.parametrize(
643 - "model_name",
644 - [MODEL_NAME],
645 -)
646 -async def test_tokenize(client: openai.AsyncOpenAI, model_name: str):
647 - base_url = str(client.base_url)[:-3].strip("/")
648 - tokenizer = get_tokenizer(tokenizer_name=model_name, tokenizer_mode="fast")
649 -
650 - for add_special in [False, True]:
651 - prompt = "This is a test prompt."
652 - tokens = tokenizer.encode(prompt, add_special_tokens=add_special)
653 -
654 - response = requests.post(base_url + "/tokenize",
655 - json={
656 - "add_special_tokens": add_special,
657 - "model": model_name,
658 - "prompt": prompt
659 - })
660 - response.raise_for_status()
661 - assert response.json() == {
662 - "tokens": tokens,
663 - "count": len(tokens),
664 - "max_model_len": 8192
665 - }
666 -
667 -
668 -@pytest.mark.asyncio
669 -@pytest.mark.parametrize(
670 - "model_name",
671 - [MODEL_NAME],
672 -)
673 -async def test_detokenize(client: openai.AsyncOpenAI, model_name: str):
674 - base_url = str(client.base_url)[:-3]
675 - tokenizer = get_tokenizer(tokenizer_name=model_name, tokenizer_mode="fast")
676 -
677 - prompt = "This is a test prompt."
678 - tokens = tokenizer.encode(prompt, add_special_tokens=False)
679 -
680 - response = requests.post(base_url + "detokenize",
681 - json={
682 - "model": model_name,
683 - "tokens": tokens
684 - })
685 - response.raise_for_status()
686 - assert response.json() == {"prompt": prompt}
tests/entrypoints/openai/test_tokenization.py new
+128
@@ -0,0 +1,128 @@
1 +import openai # use the official client for correctness check
2 +import pytest
3 +import requests
4 +
5 +from vllm.transformers_utils.tokenizer import get_tokenizer
6 +
7 +from ...utils import RemoteOpenAIServer
8 +
9 +# any model with a chat template should work here
10 +MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
11 +
12 +
13 +@pytest.fixture(scope="module")
14 +def server():
15 + with RemoteOpenAIServer([
16 + "--model",
17 + MODEL_NAME,
18 + # use half precision for speed and memory savings in CI environment
19 + "--dtype",
20 + "bfloat16",
21 + "--max-model-len",
22 + "8192",
23 + "--enforce-eager",
24 + "--max-num-seqs",
25 + "128",
26 + ]) as remote_server:
27 + yield remote_server
28 +
29 +
30 +@pytest.fixture(scope="module")
31 +def client(server):
32 + return server.get_async_client()
33 +
34 +
35 +@pytest.mark.asyncio
36 +@pytest.mark.parametrize(
37 + "model_name",
38 + [MODEL_NAME],
39 +)
40 +async def test_tokenize_completions(client: openai.AsyncOpenAI,
41 + model_name: str):
42 + base_url = str(client.base_url)[:-3].strip("/")
43 + tokenizer = get_tokenizer(tokenizer_name=model_name, tokenizer_mode="fast")
44 +
45 + for add_special in [False, True]:
46 + prompt = "This is a test prompt."
47 + tokens = tokenizer.encode(prompt, add_special_tokens=add_special)
48 +
49 + response = requests.post(base_url + "/tokenize",
50 + json={
51 + "add_special_tokens": add_special,
52 + "model": model_name,
53 + "prompt": prompt
54 + })
55 + response.raise_for_status()
56 +
57 + assert response.json() == {
58 + "tokens": tokens,
59 + "count": len(tokens),
60 + "max_model_len": 8192
61 + }
62 +
63 +
64 +@pytest.mark.asyncio
65 +@pytest.mark.parametrize(
66 + "model_name",
67 + [MODEL_NAME],
68 +)
69 +async def test_tokenize_chat(client: openai.AsyncOpenAI, model_name: str):
70 + base_url = str(client.base_url)[:-3].strip("/")
71 + tokenizer = get_tokenizer(tokenizer_name=model_name, tokenizer_mode="fast")
72 +
73 + for add_generation in [False, True]:
74 + for add_special in [False, True]:
75 + conversation = [{
76 + "role": "user",
77 + "content": "Hi there!"
78 + }, {
79 + "role": "assistant",
80 + "content": "Nice to meet you!"
81 + }, {
82 + "role": "user",
83 + "content": "Can I ask a question?"
84 + }]
85 +
86 + prompt = tokenizer.apply_chat_template(
87 + add_generation_prompt=add_generation,
88 + conversation=conversation,
89 + tokenize=False)
90 + tokens = tokenizer.encode(prompt, add_special_tokens=add_special)
91 +
92 + response = requests.post(base_url + "/tokenize",
93 + json={
94 + "add_generation_prompt":
95 + add_generation,
96 + "add_special_tokens": add_special,
97 + "messages": conversation,
98 + "model": model_name
99 + })
100 + response.raise_for_status()
101 +
102 + assert response.json() == {
103 + "tokens": tokens,
104 + "count": len(tokens),
105 + "max_model_len": 8192
106 + }
107 +
108 +
109 +@pytest.mark.asyncio
110 +@pytest.mark.parametrize(
111 + "model_name",
112 + [MODEL_NAME],
113 +)
114 +async def test_detokenize(client: openai.AsyncOpenAI, model_name: str):
115 + base_url = str(client.base_url)[:-3].strip("/")
116 + tokenizer = get_tokenizer(tokenizer_name=model_name, tokenizer_mode="fast")
117 +
118 + prompt = "This is a test prompt."
119 + tokens = tokenizer.encode(prompt, add_special_tokens=False)
120 +
121 + response = requests.post(base_url + "/detokenize",
122 + json={
123 + "model": model_name,
124 + "tokens": tokens
125 + })
126 + response.raise_for_status()
127 +
128 + assert response.json() == {"prompt": prompt}
vllm/entrypoints/openai/api_server.py
+8 -2
@@ -33,6 +33,8 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
33 from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
34 from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
35 from vllm.entrypoints.openai.serving_embedding import OpenAIServingEmbedding
36 +from vllm.entrypoints.openai.serving_tokenization import (
37 + OpenAIServingTokenization)
38 from vllm.logger import init_logger
39 from vllm.usage.usage_lib import UsageContext
40 from vllm.utils import FlexibleArgumentParser
@@ -46,6 +48,7 @@ engine_args: AsyncEngineArgs
48 openai_serving_chat: OpenAIServingChat
49 openai_serving_completion: OpenAIServingCompletion
50 openai_serving_embedding: OpenAIServingEmbedding
51 +openai_serving_tokenization: OpenAIServingTokenization
52
53 logger = init_logger('vllm.entrypoints.openai.api_server')
54
@@ -86,7 +89,7 @@ async def health() -> Response:
89
90 @router.post("/tokenize")
91 async def tokenize(request: TokenizeRequest):
89 - generator = await openai_serving_completion.create_tokenize(request)
92 + generator = await openai_serving_tokenization.create_tokenize(request)
93 if isinstance(generator, ErrorResponse):
94 return JSONResponse(content=generator.model_dump(),
95 status_code=generator.code)
@@ -97,7 +100,7 @@ async def tokenize(request: TokenizeRequest):
100
101 @router.post("/detokenize")
102 async def detokenize(request: DetokenizeRequest):
100 - generator = await openai_serving_completion.create_detokenize(request)
103 + generator = await openai_serving_tokenization.create_detokenize(request)
104 if isinstance(generator, ErrorResponse):
105 return JSONResponse(content=generator.model_dump(),
106 status_code=generator.code)
@@ -241,6 +244,7 @@ def run_server(args, llm_engine=None):
244 global openai_serving_chat
245 global openai_serving_completion
246 global openai_serving_embedding
247 + global openai_serving_tokenization
248
249 openai_serving_chat = OpenAIServingChat(engine, model_config,
250 served_model_names,
@@ -252,6 +256,8 @@ def run_server(args, llm_engine=None):
256 args.prompt_adapters)
257 openai_serving_embedding = OpenAIServingEmbedding(engine, model_config,
258 served_model_names)
259 + openai_serving_tokenization = OpenAIServingTokenization(
260 + engine, model_config, served_model_names, args.chat_template)
261 app.root_path = args.root_path
262
263 logger.info("Available routes are:")
vllm/entrypoints/openai/chat_utils.py new
+156
@@ -0,0 +1,156 @@
1 +import codecs
2 +from dataclasses import dataclass, field
3 +from functools import lru_cache
4 +from typing import Awaitable, Iterable, List, Optional, TypedDict, cast, final
5 +
6 +from openai.types.chat import (ChatCompletionContentPartImageParam,
7 + ChatCompletionContentPartTextParam)
8 +
9 +from vllm.entrypoints.openai.protocol import (ChatCompletionContentPartParam,
10 + ChatCompletionMessageParam)
11 +from vllm.entrypoints.openai.serving_engine import OpenAIServing
12 +from vllm.logger import init_logger
13 +from vllm.multimodal import MultiModalDataDict
14 +from vllm.multimodal.utils import async_get_and_parse_image
15 +
16 +logger = init_logger(__name__)
17 +
18 +
19 +@final # So that it should be compatible with Dict[str, str]
20 +class ConversationMessage(TypedDict):
21 + role: str
22 + content: str
23 +
24 +
25 +@dataclass(frozen=True)
26 +class ChatMessageParseResult:
27 + messages: List[ConversationMessage]
28 + mm_futures: List[Awaitable[MultiModalDataDict]] = field(
29 + default_factory=list)
30 +
31 +
32 +def load_chat_template(engine: OpenAIServing, chat_template: Optional[str]):
33 + tokenizer = engine.tokenizer
34 +
35 + if chat_template is not None:
36 + try:
37 + with open(chat_template, "r") as f:
38 + tokenizer.chat_template = f.read()
39 + except OSError as e:
40 + JINJA_CHARS = "{}\n"
41 + if not any(c in chat_template for c in JINJA_CHARS):
42 + msg = (f"The supplied chat template ({chat_template}) "
43 + f"looks like a file path, but it failed to be "
44 + f"opened. Reason: {e}")
45 + raise ValueError(msg) from e
46 +
47 + # If opening a file fails, set chat template to be args to
48 + # ensure we decode so our escape are interpreted correctly
49 + tokenizer.chat_template = codecs.decode(chat_template,
50 + "unicode_escape")
51 +
52 + logger.info("Using supplied chat template:\n%s",
53 + tokenizer.chat_template)
54 + elif tokenizer.chat_template is not None:
55 + logger.info("Using default chat template:\n%s",
56 + tokenizer.chat_template)
57 + else:
58 + logger.warning("No chat template provided. Chat API will not work.")
59 +
60 +
61 +@lru_cache(maxsize=None)
62 +def _image_token_str(engine: OpenAIServing) -> Optional[str]:
63 + # TODO: Let user specify how to insert image tokens into prompt
64 + # (similar to chat template)
65 + model_type = engine.model_config.hf_config.model_type
66 + if model_type == "phi3_v":
67 + # Workaround since this token is not defined in the tokenizer
68 + return "<|image_1|>"
69 + if model_type in ("blip-2", "chatglm", "fuyu", "minicpmv", "paligemma"):
70 + # These models do not use image tokens in the prompt
71 + return None
72 + if model_type.startswith("llava"):
73 + return engine.tokenizer.decode(
74 + engine.model_config.hf_config.image_token_index)
75 +
76 + else:
77 + raise TypeError("Unknown model type: {model_type}")
78 +
79 +
80 +# TODO: Let user specify how to insert image tokens into prompt
81 +# (similar to chat template)
82 +def _get_full_image_text_prompt(engine: OpenAIServing, image_token_str: str,
83 + text_prompt: str) -> str:
84 + """Combine image and text prompts for vision language model"""
85 +
86 + # NOTE: For now we assume all model architectures use the same
87 + # image + text prompt format. This may change in the future.
88 + return f"{image_token_str}\n{text_prompt}"
89 +
90 +
91 +def _parse_chat_message_content_parts(
92 + engine: OpenAIServing,
93 + role: str,
94 + parts: Iterable[ChatCompletionContentPartParam],
95 +) -> ChatMessageParseResult:
96 + texts: List[str] = []
97 + mm_futures: List[Awaitable[MultiModalDataDict]] = []
98 +
99 + for part in parts:
100 + part_type = part["type"]
101 + if part_type == "text":
102 + text = cast(ChatCompletionContentPartTextParam, part)["text"]
103 + texts.append(text)
104 + elif part_type == "image_url":
105 + if len(mm_futures) > 0:
106 + raise NotImplementedError(
107 + "Multiple 'image_url' input is currently not supported.")
108 +
109 + image_url = cast(ChatCompletionContentPartImageParam,
110 + part)["image_url"]
111 +
112 + if image_url.get("detail", "auto") != "auto":
113 + logger.warning(
114 + "'image_url.detail' is currently not supported and "
115 + "will be ignored.")
116 +
117 + image_future = async_get_and_parse_image(image_url["url"])
118 + mm_futures.append(image_future)
119 + else:
120 + raise NotImplementedError(f"Unknown part type: {part_type}")
121 +
122 + text_prompt = "\n".join(texts)
123 +
124 + if mm_futures:
125 + image_token_str = _image_token_str(engine)
126 + if image_token_str is not None:
127 + if image_token_str in text_prompt:
128 + logger.warning(
129 + "Detected image token string in the text prompt. "
130 + "Skipping prompt formatting.")
131 + else:
132 + text_prompt = _get_full_image_text_prompt(
133 + engine,
134 + image_token_str=image_token_str,
135 + text_prompt=text_prompt,
136 + )
137 +
138 + messages = [ConversationMessage(role=role, content=text_prompt)]
139 +
140 + return ChatMessageParseResult(messages=messages, mm_futures=mm_futures)
141 +
142 +
143 +def parse_chat_message_content(
144 + engine: OpenAIServing,
145 + message: ChatCompletionMessageParam,
146 +) -> ChatMessageParseResult:
147 + role = message["role"]
148 + content = message.get("content")
149 +
150 + if content is None:
151 + return ChatMessageParseResult(messages=[], mm_futures=[])
152 + if isinstance(content, str):
153 + messages = [ConversationMessage(role=role, content=content)]
154 + return ChatMessageParseResult(messages=messages, mm_futures=[])
155 +
156 + return _parse_chat_message_content_parts(engine, role, content)
vllm/entrypoints/openai/protocol.py
+5 -3
@@ -738,15 +738,17 @@ class BatchRequestOutput(OpenAIBaseModel):
738
739
740 class TokenizeRequest(OpenAIBaseModel):
741 + add_generation_prompt: bool = Field(default=True)
742 + add_special_tokens: bool = Field(default=False)
743 + prompt: Optional[str] = Field(default=None)
744 + messages: Optional[List[ChatCompletionMessageParam]] = Field(default=None)
745 model: str
742 - prompt: str
743 - add_special_tokens: bool = Field(default=True)
746
747
748 class TokenizeResponse(OpenAIBaseModel):
747 - tokens: List[int]
749 count: int
750 max_model_len: int
751 + tokens: List[int]
752
753
754 class DetokenizeRequest(OpenAIBaseModel):
vllm/entrypoints/openai/serving_chat.py
+10 -151
@@ -1,22 +1,19 @@
1 -import codecs
1 import time
3 -from dataclasses import dataclass, field
4 -from functools import cached_property
5 -from typing import (AsyncGenerator, AsyncIterator, Awaitable, Dict, Iterable,
6 - List, Optional)
2 +from typing import (AsyncGenerator, AsyncIterator, Awaitable, Dict, List,
3 + Optional)
4 from typing import Sequence as GenericSequence
8 -from typing import TypedDict, Union, cast, final
5 +from typing import Union
6
7 from fastapi import Request
11 -from openai.types.chat import (ChatCompletionContentPartImageParam,
12 - ChatCompletionContentPartTextParam)
8
9 from vllm.config import ModelConfig
10 from vllm.engine.async_llm_engine import AsyncLLMEngine
11 +from vllm.entrypoints.openai.chat_utils import (ConversationMessage,
12 + load_chat_template,
13 + parse_chat_message_content)
14 from vllm.entrypoints.openai.protocol import (
17 - ChatCompletionContentPartParam, ChatCompletionLogProb,
18 - ChatCompletionLogProbs, ChatCompletionLogProbsContent,
19 - ChatCompletionMessageParam, ChatCompletionNamedToolChoiceParam,
15 + ChatCompletionLogProb, ChatCompletionLogProbs,
16 + ChatCompletionLogProbsContent, ChatCompletionNamedToolChoiceParam,
17 ChatCompletionRequest, ChatCompletionResponse,
18 ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice,
19 ChatCompletionStreamResponse, ChatMessage, DeltaMessage, ErrorResponse,
@@ -28,7 +25,6 @@ from vllm.logger import init_logger
25 from vllm.model_executor.guided_decoding import (
26 get_guided_decoding_logits_processor)
27 from vllm.multimodal import MultiModalDataDict
31 -from vllm.multimodal.utils import async_get_and_parse_image
28 from vllm.outputs import RequestOutput
29 from vllm.sequence import Logprob
30 from vllm.tracing import (contains_trace_headers, extract_trace_headers,
@@ -38,19 +34,6 @@ from vllm.utils import random_uuid
34 logger = init_logger(__name__)
35
36
41 -@final # So that it should be compatible with Dict[str, str]
42 -class ConversationMessage(TypedDict):
43 - role: str
44 - content: str
45 -
46 -
47 -@dataclass(frozen=True)
48 -class ChatMessageParseResult:
49 - messages: List[ConversationMessage]
50 - mm_futures: List[Awaitable[MultiModalDataDict]] = field(
51 - default_factory=list)
52 -
53 -
37 class OpenAIServingChat(OpenAIServing):
38
39 def __init__(self,
@@ -66,131 +49,7 @@ class OpenAIServingChat(OpenAIServing):
49 lora_modules=lora_modules)
50
51 self.response_role = response_role
69 - self._load_chat_template(chat_template)
70 -
71 - def _load_chat_template(self, chat_template: Optional[str]):
72 - tokenizer = self.tokenizer
73 -
74 - if chat_template is not None:
75 - try:
76 - with open(chat_template, "r") as f:
77 - tokenizer.chat_template = f.read()
78 - except OSError as e:
79 - JINJA_CHARS = "{}\n"
80 - if not any(c in chat_template for c in JINJA_CHARS):
81 - msg = (f"The supplied chat template ({chat_template}) "
82 - f"looks like a file path, but it failed to be "
83 - f"opened. Reason: {e}")
84 - raise ValueError(msg) from e
85 -
86 - # If opening a file fails, set chat template to be args to
87 - # ensure we decode so our escape are interpreted correctly
88 - tokenizer.chat_template = codecs.decode(
89 - chat_template, "unicode_escape")
90 -
91 - logger.info("Using supplied chat template:\n%s",
92 - tokenizer.chat_template)
93 - elif tokenizer.chat_template is not None:
94 - logger.info("Using default chat template:\n%s",
95 - tokenizer.chat_template)
96 - else:
97 - logger.warning(
98 - "No chat template provided. Chat API will not work.")
99 -
100 - @cached_property
101 - def image_token_str(self) -> Optional[str]:
102 - # TODO: Let user specify how to insert image tokens into prompt
103 - # (similar to chat template)
104 - model_type = self.model_config.hf_config.model_type
105 - if model_type == "phi3_v":
106 - # Workaround since this token is not defined in the tokenizer
107 - return "<|image_1|>"
108 - if model_type in ("blip-2", "chatglm", "fuyu", "minicpmv",
109 - "paligemma"):
110 - # These models do not use image tokens in the prompt
111 - return None
112 - if model_type.startswith("llava"):
113 - return self.tokenizer.decode(
114 - self.model_config.hf_config.image_token_index)
115 -
116 - else:
117 - raise TypeError("Unknown model type: {model_type}")
118 -
119 - # TODO: Let user specify how to insert image tokens into prompt
120 - # (similar to chat template)
121 - def _get_full_image_text_prompt(self, image_token_str: str,
122 - text_prompt: str) -> str:
123 - """Combine image and text prompts for vision language model"""
124 -
125 - # NOTE: For now we assume all model architectures use the same
126 - # image + text prompt format. This may change in the future.
127 - return f"{image_token_str}\n{text_prompt}"
128 -
129 - def _parse_chat_message_content_parts(
130 - self,
131 - role: str,
132 - parts: Iterable[ChatCompletionContentPartParam],
133 - ) -> ChatMessageParseResult:
134 - texts: List[str] = []
135 - mm_futures: List[Awaitable[MultiModalDataDict]] = []
136 -
137 - for part in parts:
138 - part_type = part["type"]
139 - if part_type == "text":
140 - text = cast(ChatCompletionContentPartTextParam, part)["text"]
141 - texts.append(text)
142 - elif part_type == "image_url":
143 - if len(mm_futures) > 0:
144 - raise NotImplementedError(
145 - "Multiple 'image_url' input is currently not supported."
146 - )
147 -
148 - image_url = cast(ChatCompletionContentPartImageParam,
149 - part)["image_url"]
150 -
151 - if image_url.get("detail", "auto") != "auto":
152 - logger.warning(
153 - "'image_url.detail' is currently not supported and "
154 - "will be ignored.")
155 -
156 - image_future = async_get_and_parse_image(image_url["url"])
157 - mm_futures.append(image_future)
158 - else:
159 - raise NotImplementedError(f"Unknown part type: {part_type}")
160 -
161 - text_prompt = "\n".join(texts)
162 -
163 - if mm_futures:
164 - image_token_str = self.image_token_str
165 - if image_token_str is not None:
166 - if image_token_str in text_prompt:
167 - logger.warning(
168 - "Detected image token string in the text prompt. "
169 - "Skipping prompt formatting.")
170 - else:
171 - text_prompt = self._get_full_image_text_prompt(
172 - image_token_str=image_token_str,
173 - text_prompt=text_prompt,
174 - )
175 -
176 - messages = [ConversationMessage(role=role, content=text_prompt)]
177 -
178 - return ChatMessageParseResult(messages=messages, mm_futures=mm_futures)
179 -
180 - def _parse_chat_message_content(
181 - self,
182 - message: ChatCompletionMessageParam,
183 - ) -> ChatMessageParseResult:
184 - role = message["role"]
185 - content = message.get("content")
186 -
187 - if content is None:
188 - return ChatMessageParseResult(messages=[], mm_futures=[])
189 - if isinstance(content, str):
190 - messages = [ConversationMessage(role=role, content=content)]
191 - return ChatMessageParseResult(messages=messages, mm_futures=[])
192 -
193 - return self._parse_chat_message_content_parts(role, content)
52 + load_chat_template(self, chat_template)
53
54 async def create_chat_completion(
55 self,
@@ -216,7 +75,7 @@ class OpenAIServingChat(OpenAIServing):
75 mm_futures: List[Awaitable[MultiModalDataDict]] = []
76
77 for msg in request.messages:
219 - chat_parsed_result = self._parse_chat_message_content(msg)
78 + chat_parsed_result = parse_chat_message_content(self, msg)
79
80 conversation.extend(chat_parsed_result.messages)
81 mm_futures.extend(chat_parsed_result.mm_futures)
vllm/entrypoints/openai/serving_completion.py
+1 -30
@@ -16,10 +16,7 @@ from vllm.entrypoints.openai.protocol import (CompletionLogProbs,
16 CompletionResponseChoice,
17 CompletionResponseStreamChoice,
18 CompletionStreamResponse,
19 - DetokenizeRequest,
20 - DetokenizeResponse,
21 - TokenizeRequest,
22 - TokenizeResponse, UsageInfo)
19 + UsageInfo)
20 # yapf: enable
21 from vllm.entrypoints.openai.serving_engine import (LoRAModulePath,
22 OpenAIServing,
@@ -457,29 +454,3 @@ class OpenAIServingCompletion(OpenAIServing):
454 tokens=out_tokens,
455 top_logprobs=out_top_logprobs,
456 )
460 -
461 - async def create_tokenize(self,
462 - request: TokenizeRequest) -> TokenizeResponse:
463 - error_check_ret = await self._check_model(request)
464 - if error_check_ret is not None:
465 - return error_check_ret
466 -
467 - (input_ids, input_text) = self._validate_prompt_and_tokenize(
468 - request,
469 - prompt=request.prompt,
470 - add_special_tokens=request.add_special_tokens)
471 -
472 - return TokenizeResponse(tokens=input_ids,
473 - count=len(input_ids),
474 - max_model_len=self.max_model_len)
475 -
476 - async def create_detokenize(
477 - self, request: DetokenizeRequest) -> DetokenizeResponse:
478 - error_check_ret = await self._check_model(request)
479 - if error_check_ret is not None:
480 - return error_check_ret
481 -
482 - (input_ids, input_text) = self._validate_prompt_and_tokenize(
483 - request, prompt_ids=request.tokens)
484 -
485 - return DetokenizeResponse(prompt=input_text)
vllm/entrypoints/openai/serving_tokenization.py new
+73
@@ -0,0 +1,73 @@
1 +from typing import List, Optional
2 +
3 +from vllm.config import ModelConfig
4 +from vllm.engine.async_llm_engine import AsyncLLMEngine
5 +from vllm.entrypoints.openai.chat_utils import (ConversationMessage,
6 + load_chat_template,
7 + parse_chat_message_content)
8 +from vllm.entrypoints.openai.protocol import (DetokenizeRequest,
9 + DetokenizeResponse,
10 + TokenizeRequest,
11 + TokenizeResponse)
12 +from vllm.entrypoints.openai.serving_engine import OpenAIServing
13 +
14 +
15 +class OpenAIServingTokenization(OpenAIServing):
16 +
17 + def __init__(self,
18 + engine: AsyncLLMEngine,
19 + model_config: ModelConfig,
20 + served_model_names: List[str],
21 + chat_template: Optional[str] = None):
22 + super().__init__(engine=engine,
23 + model_config=model_config,
24 + served_model_names=served_model_names,
25 + lora_modules=None)
26 +
27 + load_chat_template(self, chat_template)
28 +
29 + async def create_tokenize(self,
30 + request: TokenizeRequest) -> TokenizeResponse:
31 + error_check_ret = await self._check_model(request)
32 + if error_check_ret is not None:
33 + return error_check_ret
34 +
35 + if not (request.prompt or request.messages):
36 + return self.create_error_response(
37 + "Either `prompt` or `messages` should be provided.")
38 +
39 + if (request.prompt and request.messages):
40 + return self.create_error_response(
41 + "Only one of `prompt` or `messages` should be provided.")
42 +
43 + if request.messages:
44 + conversation: List[ConversationMessage] = []
45 +
46 + for message in request.messages:
47 + conversation.extend(
48 + parse_chat_message_content(self, message).messages)
49 +
50 + request.prompt = self.tokenizer.apply_chat_template(
51 + add_generation_prompt=request.add_generation_prompt,
52 + conversation=conversation,
53 + tokenize=False)
54 +
55 + (input_ids, input_text) = self._validate_prompt_and_tokenize(
56 + request,
57 + prompt=request.prompt,
58 + add_special_tokens=request.add_special_tokens)
59 +
60 + return TokenizeResponse(tokens=input_ids,
61 + count=len(input_ids),
62 + max_model_len=self.max_model_len)
63 +
64 + async def create_detokenize(
65 + self, request: DetokenizeRequest) -> DetokenizeResponse:
66 + error_check_ret = await self._check_model(request)
67 + if error_check_ret is not None:
68 + return error_check_ret
69 +
70 + (input_ids, input_text) = self._validate_prompt_and_tokenize(
71 + request, prompt_ids=request.tokens)
72 +
73 + return DetokenizeResponse(prompt=input_text)