A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[BugFix][Frontend] Use LoRA tokenizer in OpenAI APIs (#6227)
Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
Nick Hill committed
Jul 18, 2024 at 00:13 UTC
e2fbaee7258810bcd43725e9ca7f1444a88f91f3
16 files changed
+267
-186
tests/async_engine/test_chat_template.py
+6
-27
@@ -1,6 +1,5 @@
1
import os
2
import pathlib
3
-from dataclasses import dataclass
3
4
import pytest
5
@@ -50,23 +49,9 @@ TEST_MESSAGES = [
49
]
50
51
53
-@dataclass
54
-class MockTokenizer:
55
- chat_template = None
56
-
57
-
58
-@dataclass
59
-class MockServingChat:
60
- tokenizer: MockTokenizer
61
-
62
-
52
def test_load_chat_template():
53
# Testing chatml template
65
- tokenizer = MockTokenizer()
66
- mock_serving_chat = MockServingChat(tokenizer)
67
- load_chat_template(mock_serving_chat, chat_template=chatml_jinja_path)
68
-
69
- template_content = tokenizer.chat_template
54
+ template_content = load_chat_template(chat_template=chatml_jinja_path)
55
56
# Test assertions
57
assert template_content is not None
@@ -78,22 +63,16 @@ def test_load_chat_template():
63
def test_no_load_chat_template_filelike():
64
# Testing chatml template
65
template = "../../examples/does_not_exist"
81
- tokenizer = MockTokenizer()
82
-
83
- mock_serving_chat = MockServingChat(tokenizer)
66
67
with pytest.raises(ValueError, match="looks like a file path"):
86
- load_chat_template(mock_serving_chat, chat_template=template)
68
+ load_chat_template(chat_template=template)
69
70
71
def test_no_load_chat_template_literallike():
72
# Testing chatml template
73
template = "{{ messages }}"
92
- tokenizer = MockTokenizer()
74
94
- mock_serving_chat = MockServingChat(tokenizer)
95
- load_chat_template(mock_serving_chat, chat_template=template)
96
- template_content = tokenizer.chat_template
75
+ template_content = load_chat_template(chat_template=template)
76
77
assert template_content == template
78
@@ -105,8 +84,7 @@ def test_get_gen_prompt(model, template, add_generation_prompt,
84
expected_output):
85
# Initialize the tokenizer
86
tokenizer = get_tokenizer(tokenizer_name=model)
108
- mock_serving_chat = MockServingChat(tokenizer)
109
- load_chat_template(mock_serving_chat, chat_template=template)
87
+ template_content = load_chat_template(chat_template=template)
88
89
# Create a mock request object using keyword arguments
90
mock_request = ChatCompletionRequest(
@@ -118,7 +96,8 @@ def test_get_gen_prompt(model, template, add_generation_prompt,
96
result = tokenizer.apply_chat_template(
97
conversation=mock_request.messages,
98
tokenize=False,
121
- add_generation_prompt=mock_request.add_generation_prompt)
99
+ add_generation_prompt=mock_request.add_generation_prompt,
100
+ chat_template=mock_request.chat_template or template_content)
101
102
# Test assertion
103
assert result == expected_output, (
tests/entrypoints/openai/test_chat.py
+4
-9
@@ -7,11 +7,11 @@ import jsonschema
7
import openai # use the official client for correctness check
8
import pytest
9
import torch
10
-# downloading lora to test lora requests
11
-from huggingface_hub import snapshot_download
10
from openai import BadRequestError
11
12
from ...utils import RemoteOpenAIServer
13
+from .test_completion import zephyr_lora_added_tokens_files # noqa: F401
14
+from .test_completion import zephyr_lora_files # noqa: F401
15
16
# any model with a chat template should work here
17
MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
@@ -21,12 +21,7 @@ LORA_NAME = "typeof/zephyr-7b-beta-lora"
21
22
23
@pytest.fixture(scope="module")
24
-def zephyr_lora_files():
25
- return snapshot_download(repo_id=LORA_NAME)
26
-
27
-
28
-@pytest.fixture(scope="module")
29
-def server(zephyr_lora_files):
24
+def server(zephyr_lora_files, zephyr_lora_added_tokens_files): # noqa: F811
25
args = [
26
# use half precision for speed and memory savings in CI environment
27
"--dtype",
@@ -38,7 +33,7 @@ def server(zephyr_lora_files):
33
"--enable-lora",
34
"--lora-modules",
35
f"zephyr-lora={zephyr_lora_files}",
41
- f"zephyr-lora2={zephyr_lora_files}",
36
+ f"zephyr-lora2={zephyr_lora_added_tokens_files}",
37
"--max-lora-rank",
38
"64",
39
"--max-cpu-loras",
tests/entrypoints/openai/test_completion.py
+49
-2
@@ -1,6 +1,8 @@
1
# imports for guided decoding tests
2
import json
3
import re
4
+import shutil
5
+from tempfile import TemporaryDirectory
6
from typing import List
7
8
import jsonschema
@@ -9,6 +11,7 @@ import pytest
11
# downloading lora to test lora requests
12
from huggingface_hub import snapshot_download
13
from openai import BadRequestError
14
+from transformers import AutoTokenizer
15
16
from vllm.transformers_utils.tokenizer import get_tokenizer
17
@@ -30,13 +33,29 @@ def zephyr_lora_files():
33
return snapshot_download(repo_id=LORA_NAME)
34
35
36
+@pytest.fixture(scope="module")
37
+def zephyr_lora_added_tokens_files(zephyr_lora_files):
38
+ tmp_dir = TemporaryDirectory()
39
+ tmp_model_dir = f"{tmp_dir.name}/zephyr"
40
+ shutil.copytree(zephyr_lora_files, tmp_model_dir)
41
+ tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
42
+ # Copy tokenizer to adapter and add some unique tokens
43
+ # 32000, 32001, 32002
44
+ added = tokenizer.add_tokens(["vllm1", "vllm2", "vllm3"],
45
+ special_tokens=True)
46
+ assert added == 3
47
+ tokenizer.save_pretrained(tmp_model_dir)
48
+ yield tmp_model_dir
49
+ tmp_dir.cleanup()
50
+
51
+
52
@pytest.fixture(scope="module")
53
def zephyr_pa_files():
54
return snapshot_download(repo_id=PA_NAME)
55
56
57
@pytest.fixture(scope="module")
39
-def server(zephyr_lora_files, zephyr_pa_files):
58
+def server(zephyr_lora_files, zephyr_lora_added_tokens_files, zephyr_pa_files):
59
args = [
60
# use half precision for speed and memory savings in CI environment
61
"--dtype",
@@ -50,7 +69,7 @@ def server(zephyr_lora_files, zephyr_pa_files):
69
"--enable-lora",
70
"--lora-modules",
71
f"zephyr-lora={zephyr_lora_files}",
53
- f"zephyr-lora2={zephyr_lora_files}",
72
+ f"zephyr-lora2={zephyr_lora_added_tokens_files}",
73
"--max-lora-rank",
74
"64",
75
"--max-cpu-loras",
@@ -111,6 +130,34 @@ async def test_single_completion(client: openai.AsyncOpenAI, model_name: str,
130
assert len(completion.choices[0].text) >= 1
131
132
133
+@pytest.mark.asyncio
134
+async def test_added_lora_tokens(client: openai.AsyncOpenAI):
135
+ # test using token IDs
136
+ completion = await client.completions.create(
137
+ model="zephyr-lora2",
138
+ prompt=[0, 0, 32000, 32001, 32002],
139
+ echo=True,
140
+ max_tokens=5,
141
+ temperature=0.0,
142
+ )
143
+ # Added tokens should appear in tokenized prompt
144
+ assert completion.choices[0].text.startswith("<unk><unk>vllm1vllm2vllm3")
145
+
146
+
147
+@pytest.mark.asyncio
148
+async def test_added_lora_tokens_base_model(client: openai.AsyncOpenAI):
149
+ # test using token IDs
150
+ completion = await client.completions.create(
151
+ model=MODEL_NAME,
152
+ prompt=[0, 0, 32000, 32001, 32002],
153
+ echo=True,
154
+ max_tokens=5,
155
+ temperature=0.0,
156
+ )
157
+ # Added tokens should not appear in tokenized prompt
158
+ assert "vllm" not in completion.choices[0].text
159
+
160
+
161
@pytest.mark.asyncio
162
@pytest.mark.parametrize(
163
# first test base model, then test loras, then test prompt adapters
tests/entrypoints/openai/test_serving_chat.py
+1
-2
@@ -38,5 +38,4 @@ async def _async_serving_chat_init():
38
39
def test_async_serving_chat_init():
40
serving_completion = asyncio.run(_async_serving_chat_init())
41
- assert serving_completion.tokenizer is not None
42
- assert serving_completion.tokenizer.chat_template == CHAT_TEMPLATE
41
+ assert serving_completion.chat_template == CHAT_TEMPLATE
tests/entrypoints/openai/test_tokenization.py
+40
-16
@@ -5,13 +5,15 @@ import requests
5
from vllm.transformers_utils.tokenizer import get_tokenizer
6
7
from ...utils import RemoteOpenAIServer
8
+from .test_completion import zephyr_lora_added_tokens_files # noqa: F401
9
+from .test_completion import zephyr_lora_files # noqa: F401
10
11
# any model with a chat template should work here
12
MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
13
14
15
@pytest.fixture(scope="module")
14
-def server():
16
+def server(zephyr_lora_added_tokens_files: str): # noqa: F811
17
args = [
18
# use half precision for speed and memory savings in CI environment
19
"--dtype",
@@ -21,12 +23,25 @@ def server():
23
"--enforce-eager",
24
"--max-num-seqs",
25
"128",
26
+ # lora config
27
+ "--enable-lora",
28
+ "--lora-modules",
29
+ f"zephyr-lora2={zephyr_lora_added_tokens_files}",
30
+ "--max-lora-rank",
31
+ "64",
32
]
33
34
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
35
yield remote_server
36
37
38
+@pytest.fixture(scope="module")
39
+def tokenizer_name(model_name: str,
40
+ zephyr_lora_added_tokens_files: str): # noqa: F811
41
+ return zephyr_lora_added_tokens_files if (
42
+ model_name == "zephyr-lora2") else model_name
43
+
44
+
45
@pytest.fixture(scope="module")
46
def client(server):
47
return server.get_async_client()
@@ -34,16 +49,18 @@ def client(server):
49
50
@pytest.mark.asyncio
51
@pytest.mark.parametrize(
37
- "model_name",
38
- [MODEL_NAME],
52
+ "model_name,tokenizer_name",
53
+ [(MODEL_NAME, MODEL_NAME), ("zephyr-lora2", "zephyr-lora2")],
54
+ indirect=["tokenizer_name"],
55
)
56
async def test_tokenize_completions(client: openai.AsyncOpenAI,
41
- model_name: str):
57
+ model_name: str, tokenizer_name: str):
58
base_url = str(client.base_url)[:-3].strip("/")
43
- tokenizer = get_tokenizer(tokenizer_name=model_name, tokenizer_mode="fast")
59
+ tokenizer = get_tokenizer(tokenizer_name=tokenizer_name,
60
+ tokenizer_mode="fast")
61
62
for add_special in [False, True]:
46
- prompt = "This is a test prompt."
63
+ prompt = "vllm1 This is a test prompt."
64
tokens = tokenizer.encode(prompt, add_special_tokens=add_special)
65
66
response = requests.post(base_url + "/tokenize",
@@ -63,12 +80,15 @@ async def test_tokenize_completions(client: openai.AsyncOpenAI,
80
81
@pytest.mark.asyncio
82
@pytest.mark.parametrize(
66
- "model_name",
67
- [MODEL_NAME],
83
+ "model_name,tokenizer_name",
84
+ [(MODEL_NAME, MODEL_NAME), ("zephyr-lora2", "zephyr-lora2")],
85
+ indirect=["tokenizer_name"],
86
)
69
-async def test_tokenize_chat(client: openai.AsyncOpenAI, model_name: str):
87
+async def test_tokenize_chat(client: openai.AsyncOpenAI, model_name: str,
88
+ tokenizer_name: str):
89
base_url = str(client.base_url)[:-3].strip("/")
71
- tokenizer = get_tokenizer(tokenizer_name=model_name, tokenizer_mode="fast")
90
+ tokenizer = get_tokenizer(tokenizer_name=tokenizer_name,
91
+ tokenizer_mode="fast")
92
93
for add_generation in [False, True]:
94
for add_special in [False, True]:
@@ -80,7 +100,7 @@ async def test_tokenize_chat(client: openai.AsyncOpenAI, model_name: str):
100
"content": "Nice to meet you!"
101
}, {
102
"role": "user",
83
- "content": "Can I ask a question?"
103
+ "content": "Can I ask a question? vllm1"
104
}]
105
106
prompt = tokenizer.apply_chat_template(
@@ -108,16 +128,20 @@ async def test_tokenize_chat(client: openai.AsyncOpenAI, model_name: str):
128
129
@pytest.mark.asyncio
130
@pytest.mark.parametrize(
111
- "model_name",
112
- [MODEL_NAME],
131
+ "model_name,tokenizer_name",
132
+ [(MODEL_NAME, MODEL_NAME), ("zephyr-lora2", "zephyr-lora2")],
133
+ indirect=["tokenizer_name"],
134
)
114
-async def test_detokenize(client: openai.AsyncOpenAI, model_name: str):
135
+async def test_detokenize(client: openai.AsyncOpenAI, model_name: str,
136
+ tokenizer_name: str):
137
base_url = str(client.base_url)[:-3].strip("/")
116
- tokenizer = get_tokenizer(tokenizer_name=model_name, tokenizer_mode="fast")
138
+ tokenizer = get_tokenizer(tokenizer_name=tokenizer_name,
139
+ tokenizer_mode="fast")
140
118
- prompt = "This is a test prompt."
141
+ prompt = "This is a test prompt. vllm1"
142
tokens = tokenizer.encode(prompt, add_special_tokens=False)
143
144
+ print(f"CALLING {base_url} FOR {model_name}")
145
response = requests.post(base_url + "/detokenize",
146
json={
147
"model": model_name,
vllm/engine/async_llm_engine.py
+9
-4
@@ -480,11 +480,16 @@ class AsyncLLMEngine:
480
self.set_errored(exc)
481
self._request_tracker.propagate_exception(exc)
482
483
- async def get_tokenizer(self) -> "PreTrainedTokenizer":
483
+ async def get_tokenizer(
484
+ self,
485
+ lora_request: Optional[LoRARequest] = None,
486
+ ) -> "PreTrainedTokenizer":
487
if self.engine_use_ray:
485
- return await self.engine.get_tokenizer.remote() # type: ignore
486
- else:
487
- return self.engine.get_tokenizer()
488
+ return await self.engine.get_tokenizer.remote( # type: ignore
489
+ lora_request)
490
+
491
+ return await (self.engine.get_tokenizer_group().
492
+ get_lora_tokenizer_async(lora_request))
493
494
def start_background_loop(self) -> None:
495
"""Start the background loop."""
vllm/engine/llm_engine.py
+5
-2
@@ -455,8 +455,11 @@ class LLMEngine:
455
456
return self.tokenizer
457
458
- def get_tokenizer(self) -> "PreTrainedTokenizer":
459
- return self.get_tokenizer_group().get_lora_tokenizer(None)
458
+ def get_tokenizer(
459
+ self,
460
+ lora_request: Optional[LoRARequest] = None
461
+ ) -> "PreTrainedTokenizer":
462
+ return self.get_tokenizer_group().get_lora_tokenizer(lora_request)
463
464
def get_tokenizer_for_seq(self,
465
sequence: Sequence) -> "PreTrainedTokenizer":
vllm/entrypoints/openai/api_server.py
+2
-1
@@ -257,7 +257,8 @@ def run_server(args, llm_engine=None):
257
openai_serving_embedding = OpenAIServingEmbedding(engine, model_config,
258
served_model_names)
259
openai_serving_tokenization = OpenAIServingTokenization(
260
- engine, model_config, served_model_names, args.chat_template)
260
+ engine, model_config, served_model_names, args.lora_modules,
261
+ args.chat_template)
262
app.root_path = args.root_path
263
264
logger.info("Available routes are:")
vllm/entrypoints/openai/chat_utils.py
+33
-39
@@ -5,10 +5,11 @@ from typing import Awaitable, Iterable, List, Optional, TypedDict, cast, final
5
6
from openai.types.chat import (ChatCompletionContentPartImageParam,
7
ChatCompletionContentPartTextParam)
8
+from transformers import PreTrainedTokenizer
9
10
+from vllm.config import ModelConfig
11
from vllm.entrypoints.openai.protocol import (ChatCompletionContentPartParam,
12
ChatCompletionMessageParam)
11
-from vllm.entrypoints.openai.serving_engine import OpenAIServing
13
from vllm.logger import init_logger
14
from vllm.multimodal import MultiModalDataDict
15
from vllm.multimodal.utils import async_get_and_parse_image
@@ -29,40 +30,34 @@ class ChatMessageParseResult:
30
default_factory=list)
31
32
32
-def load_chat_template(engine: OpenAIServing, chat_template: Optional[str]):
33
- tokenizer = engine.tokenizer
34
-
35
- if chat_template is not None:
36
- try:
37
- with open(chat_template, "r") as f:
38
- tokenizer.chat_template = f.read()
39
- except OSError as e:
40
- JINJA_CHARS = "{}\n"
41
- if not any(c in chat_template for c in JINJA_CHARS):
42
- msg = (f"The supplied chat template ({chat_template}) "
43
- f"looks like a file path, but it failed to be "
44
- f"opened. Reason: {e}")
45
- raise ValueError(msg) from e
33
+def load_chat_template(chat_template: Optional[str]) -> Optional[str]:
34
+ if chat_template is None:
35
+ return None
36
+ try:
37
+ with open(chat_template, "r") as f:
38
+ resolved_chat_template = f.read()
39
+ except OSError as e:
40
+ JINJA_CHARS = "{}\n"
41
+ if not any(c in chat_template for c in JINJA_CHARS):
42
+ msg = (f"The supplied chat template ({chat_template}) "
43
+ f"looks like a file path, but it failed to be "
44
+ f"opened. Reason: {e}")
45
+ raise ValueError(msg) from e
46
47
- # If opening a file fails, set chat template to be args to
48
- # ensure we decode so our escape are interpreted correctly
49
- tokenizer.chat_template = codecs.decode(chat_template,
50
- "unicode_escape")
47
+ # If opening a file fails, set chat template to be args to
48
+ # ensure we decode so our escape are interpreted correctly
49
+ resolved_chat_template = codecs.decode(chat_template, "unicode_escape")
50
52
- logger.info("Using supplied chat template:\n%s",
53
- tokenizer.chat_template)
54
- elif tokenizer.chat_template is not None:
55
- logger.info("Using default chat template:\n%s",
56
- tokenizer.chat_template)
57
- else:
58
- logger.warning("No chat template provided. Chat API will not work.")
51
+ logger.info("Using supplied chat template:\n%s", resolved_chat_template)
52
+ return resolved_chat_template
53
54
55
@lru_cache(maxsize=None)
62
-def _image_token_str(engine: OpenAIServing) -> Optional[str]:
56
+def _image_token_str(model_config: ModelConfig,
57
+ tokenizer: PreTrainedTokenizer) -> Optional[str]:
58
# TODO: Let user specify how to insert image tokens into prompt
59
# (similar to chat template)
65
- model_type = engine.model_config.hf_config.model_type
60
+ model_type = model_config.hf_config.model_type
61
if model_type == "phi3_v":
62
# Workaround since this token is not defined in the tokenizer
63
return "<|image_1|>"
@@ -70,17 +65,14 @@ def _image_token_str(engine: OpenAIServing) -> Optional[str]:
65
# These models do not use image tokens in the prompt
66
return None
67
if model_type.startswith("llava"):
73
- return engine.tokenizer.decode(
74
- engine.model_config.hf_config.image_token_index)
68
+ return tokenizer.decode(model_config.hf_config.image_token_index)
69
76
- else:
77
- raise TypeError("Unknown model type: {model_type}")
70
+ raise TypeError("Unknown model type: {model_type}")
71
72
73
# TODO: Let user specify how to insert image tokens into prompt
74
# (similar to chat template)
82
-def _get_full_image_text_prompt(engine: OpenAIServing, image_token_str: str,
83
- text_prompt: str) -> str:
75
+def _get_full_image_text_prompt(image_token_str: str, text_prompt: str) -> str:
76
"""Combine image and text prompts for vision language model"""
77
78
# NOTE: For now we assume all model architectures use the same
@@ -89,9 +81,10 @@ def _get_full_image_text_prompt(engine: OpenAIServing, image_token_str: str,
81
82
83
def _parse_chat_message_content_parts(
92
- engine: OpenAIServing,
84
role: str,
85
parts: Iterable[ChatCompletionContentPartParam],
86
+ model_config: ModelConfig,
87
+ tokenizer: PreTrainedTokenizer,
88
) -> ChatMessageParseResult:
89
texts: List[str] = []
90
mm_futures: List[Awaitable[MultiModalDataDict]] = []
@@ -122,7 +115,7 @@ def _parse_chat_message_content_parts(
115
text_prompt = "\n".join(texts)
116
117
if mm_futures:
125
- image_token_str = _image_token_str(engine)
118
+ image_token_str = _image_token_str(model_config, tokenizer)
119
if image_token_str is not None:
120
if image_token_str in text_prompt:
121
logger.warning(
@@ -130,7 +123,6 @@ def _parse_chat_message_content_parts(
123
"Skipping prompt formatting.")
124
else:
125
text_prompt = _get_full_image_text_prompt(
133
- engine,
126
image_token_str=image_token_str,
127
text_prompt=text_prompt,
128
)
@@ -141,8 +133,9 @@ def _parse_chat_message_content_parts(
133
134
135
def parse_chat_message_content(
144
- engine: OpenAIServing,
136
message: ChatCompletionMessageParam,
137
+ model_config: ModelConfig,
138
+ tokenizer: PreTrainedTokenizer,
139
) -> ChatMessageParseResult:
140
role = message["role"]
141
content = message.get("content")
@@ -153,4 +146,5 @@ def parse_chat_message_content(
146
messages = [ConversationMessage(role=role, content=content)]
147
return ChatMessageParseResult(messages=messages, mm_futures=[])
148
156
- return _parse_chat_message_content_parts(engine, role, content)
149
+ return _parse_chat_message_content_parts(role, content, model_config,
150
+ tokenizer)
vllm/entrypoints/openai/serving_chat.py
+44
-29
@@ -5,6 +5,7 @@ from typing import Sequence as GenericSequence
5
from typing import Union
6
7
from fastapi import Request
8
+from transformers import PreTrainedTokenizer
9
10
from vllm.config import ModelConfig
11
from vllm.engine.async_llm_engine import AsyncLLMEngine
@@ -49,7 +50,9 @@ class OpenAIServingChat(OpenAIServing):
50
lora_modules=lora_modules)
51
52
self.response_role = response_role
52
- load_chat_template(self, chat_template)
53
+
54
+ # If this is None we use the tokenizer's default chat template
55
+ self.chat_template = load_chat_template(chat_template)
56
57
async def create_chat_completion(
58
self,
@@ -71,11 +74,15 @@ class OpenAIServingChat(OpenAIServing):
74
return error_check_ret
75
76
try:
77
+ _, lora_request = self._maybe_get_adapter(request)
78
+ tokenizer = await self.engine.get_tokenizer(lora_request)
79
+
80
conversation: List[ConversationMessage] = []
81
mm_futures: List[Awaitable[MultiModalDataDict]] = []
82
83
for msg in request.messages:
78
- chat_parsed_result = parse_chat_message_content(self, msg)
84
+ chat_parsed_result = parse_chat_message_content(
85
+ msg, self.model_config, tokenizer)
86
87
conversation.extend(chat_parsed_result.messages)
88
mm_futures.extend(chat_parsed_result.mm_futures)
@@ -84,13 +91,13 @@ class OpenAIServingChat(OpenAIServing):
91
tool.model_dump() for tool in request.tools
92
]
93
87
- prompt = self.tokenizer.apply_chat_template(
94
+ prompt = tokenizer.apply_chat_template(
95
conversation=conversation,
96
tokenize=False,
97
add_generation_prompt=request.add_generation_prompt,
98
tools=tool_dicts,
99
documents=request.documents,
93
- chat_template=request.chat_template,
100
+ chat_template=request.chat_template or self.chat_template,
101
**(request.chat_template_kwargs or {}),
102
)
103
except Exception as e:
@@ -112,19 +119,19 @@ class OpenAIServingChat(OpenAIServing):
119
request_id = f"cmpl-{random_uuid()}"
120
try:
121
# Tokenize/detokenize depending on prompt format (string/token list)
115
- prompt_ids, prompt_text = self._validate_prompt_and_tokenize(
122
+ prompt_ids, prompt_text = await self._validate_prompt_and_tokenize(
123
request,
124
+ tokenizer,
125
prompt=prompt,
126
add_special_tokens=request.add_special_tokens)
127
sampling_params = request.to_sampling_params()
120
- _, lora_request = self._maybe_get_adapter(request)
128
decoding_config = await self.engine.get_decoding_config()
129
guided_decoding_backend = request.guided_decoding_backend \
130
or decoding_config.guided_decoding_backend
131
guided_decode_logits_processor = (
125
- await get_guided_decoding_logits_processor(
126
- guided_decoding_backend, request, await
127
- self.engine.get_tokenizer()))
132
+ await
133
+ get_guided_decoding_logits_processor(guided_decoding_backend,
134
+ request, tokenizer))
135
if guided_decode_logits_processor:
136
if sampling_params.logits_processors is None:
137
sampling_params.logits_processors = []
@@ -158,12 +165,12 @@ class OpenAIServingChat(OpenAIServing):
165
# Streaming response
166
if request.stream:
167
return self.chat_completion_stream_generator(
161
- request, result_generator, request_id, conversation)
168
+ request, result_generator, request_id, conversation, tokenizer)
169
else:
170
try:
171
return await self.chat_completion_full_generator(
172
request, raw_request, result_generator, request_id,
166
- conversation)
173
+ conversation, tokenizer)
174
except ValueError as e:
175
# TODO: Use a vllm-specific Validation Error
176
return self.create_error_response(str(e))
@@ -175,9 +182,12 @@ class OpenAIServingChat(OpenAIServing):
182
return request.messages[-1]["role"]
183
184
async def chat_completion_stream_generator(
178
- self, request: ChatCompletionRequest,
179
- result_generator: AsyncIterator[RequestOutput], request_id: str,
180
- conversation: List[ConversationMessage]
185
+ self,
186
+ request: ChatCompletionRequest,
187
+ result_generator: AsyncIterator[RequestOutput],
188
+ request_id: str,
189
+ conversation: List[ConversationMessage],
190
+ tokenizer: PreTrainedTokenizer,
191
) -> AsyncGenerator[str, None]:
192
model_name = self.served_model_names[0]
193
created_time = int(time.time())
@@ -264,6 +274,7 @@ class OpenAIServingChat(OpenAIServing):
274
logprobs = self._create_chat_logprobs(
275
token_ids=delta_token_ids,
276
top_logprobs=out_logprobs,
277
+ tokenizer=tokenizer,
278
num_output_top_logprobs=request.top_logprobs,
279
)
280
else:
@@ -352,9 +363,13 @@ class OpenAIServingChat(OpenAIServing):
363
yield "data: [DONE]\n\n"
364
365
async def chat_completion_full_generator(
355
- self, request: ChatCompletionRequest, raw_request: Optional[Request],
356
- result_generator: AsyncIterator[RequestOutput], request_id: str,
357
- conversation: List[ConversationMessage]
366
+ self,
367
+ request: ChatCompletionRequest,
368
+ raw_request: Optional[Request],
369
+ result_generator: AsyncIterator[RequestOutput],
370
+ request_id: str,
371
+ conversation: List[ConversationMessage],
372
+ tokenizer: PreTrainedTokenizer,
373
) -> Union[ErrorResponse, ChatCompletionResponse]:
374
375
model_name = self.served_model_names[0]
@@ -382,6 +397,7 @@ class OpenAIServingChat(OpenAIServing):
397
token_ids=token_ids,
398
top_logprobs=out_logprobs,
399
num_output_top_logprobs=request.top_logprobs,
400
+ tokenizer=tokenizer,
401
)
402
else:
403
logprobs = None
@@ -436,16 +452,14 @@ class OpenAIServingChat(OpenAIServing):
452
return response
453
454
def _get_top_logprobs(
439
- self, logprobs: Dict[int, Logprob],
440
- top_logprobs: Optional[int]) -> List[ChatCompletionLogProb]:
455
+ self, logprobs: Dict[int, Logprob], top_logprobs: Optional[int],
456
+ tokenizer: PreTrainedTokenizer) -> List[ChatCompletionLogProb]:
457
return [
458
ChatCompletionLogProb(
443
- token=self._get_decoded_token(p[1], p[0]),
459
+ token=(token := self._get_decoded_token(p[1], p[0],
460
+ tokenizer)),
461
logprob=max(p[1].logprob, -9999.0),
445
- bytes=list(
446
- self._get_decoded_token(p[1],
447
- p[0]).encode("utf-8",
448
- errors="replace")))
462
+ bytes=list(token.encode("utf-8", errors="replace")))
463
for i, p in enumerate(logprobs.items())
464
if top_logprobs and i < top_logprobs
465
]
@@ -454,6 +468,7 @@ class OpenAIServingChat(OpenAIServing):
468
self,
469
token_ids: GenericSequence[int],
470
top_logprobs: GenericSequence[Optional[Dict[int, Logprob]]],
471
+ tokenizer: PreTrainedTokenizer,
472
num_output_top_logprobs: Optional[int] = None,
473
) -> ChatCompletionLogProbs:
474
"""Create OpenAI-style logprobs."""
@@ -463,12 +478,11 @@ class OpenAIServingChat(OpenAIServing):
478
for i, token_id in enumerate(token_ids):
479
step_top_logprobs = top_logprobs[i]
480
if step_top_logprobs is None:
481
+ token = tokenizer.decode(token_id)
482
logprobs_content.append(
483
ChatCompletionLogProbsContent(
468
- token=self.tokenizer.decode(token_id),
469
- bytes=list(
470
- self.tokenizer.decode(token_id).encode(
471
- "utf-8", errors="replace"))))
484
+ token=token,
485
+ bytes=list(token.encode("utf-8", errors="replace"))))
486
else:
487
logprobs_content.append(
488
ChatCompletionLogProbsContent(
@@ -479,6 +493,7 @@ class OpenAIServingChat(OpenAIServing):
493
step_top_logprobs[token_id].decoded_token.encode(
494
"utf-8", errors="replace")),
495
top_logprobs=self._get_top_logprobs(
482
- step_top_logprobs, num_output_top_logprobs)))
496
+ step_top_logprobs, num_output_top_logprobs,
497
+ tokenizer)))
498
499
return ChatCompletionLogProbs(content=logprobs_content)
vllm/entrypoints/openai/serving_completion.py
+26
-21
@@ -5,6 +5,7 @@ from typing import Sequence as GenericSequence
5
from typing import Tuple
6
7
from fastapi import Request
8
+from transformers import PreTrainedTokenizer
9
10
from vllm.config import ModelConfig
11
from vllm.engine.async_llm_engine import AsyncLLMEngine
@@ -100,20 +101,22 @@ class OpenAIServingCompletion(OpenAIServing):
101
# Schedule the request and get the result generator.
102
generators: List[AsyncIterator[RequestOutput]] = []
103
try:
103
- sampling_params = request.to_sampling_params()
104
adapter_type, adapter_request = self._maybe_get_adapter(request)
105
lora_request, prompt_adapter_request = None, None
106
if adapter_type == 'LoRA':
107
lora_request, prompt_adapter_request = adapter_request, None
108
elif adapter_type == 'PromptAdapter':
109
lora_request, prompt_adapter_request = None, adapter_request
110
+ tokenizer = await self.engine.get_tokenizer(lora_request)
111
+
112
+ sampling_params = request.to_sampling_params()
113
decoding_config = await self.engine.get_decoding_config()
114
guided_decoding_backend = request.guided_decoding_backend \
115
or decoding_config.guided_decoding_backend
116
guided_decode_logit_processor = (
114
- await get_guided_decoding_logits_processor(
115
- guided_decoding_backend, request, await
116
- self.engine.get_tokenizer()))
117
+ await
118
+ get_guided_decoding_logits_processor(guided_decoding_backend,
119
+ request, tokenizer))
120
if guided_decode_logit_processor is not None:
121
if sampling_params.logits_processors is None:
122
sampling_params.logits_processors = []
@@ -122,18 +125,13 @@ class OpenAIServingCompletion(OpenAIServing):
125
prompt_is_tokens, prompts = parse_prompt_format(request.prompt)
126
127
for i, prompt in enumerate(prompts):
125
- if prompt_is_tokens:
126
- prompt_formats = self._validate_prompt_and_tokenize(
127
- request,
128
- prompt_ids=prompt,
129
- truncate_prompt_tokens=sampling_params.
130
- truncate_prompt_tokens)
131
- else:
132
- prompt_formats = self._validate_prompt_and_tokenize(
133
- request,
134
- prompt=prompt,
135
- truncate_prompt_tokens=sampling_params.
136
- truncate_prompt_tokens)
128
+ prompt_arg = "prompt_ids" if prompt_is_tokens else "prompt"
129
+ prompt_formats = await self._validate_prompt_and_tokenize(
130
+ request,
131
+ tokenizer,
132
+ truncate_prompt_tokens=sampling_params.
133
+ truncate_prompt_tokens,
134
+ **{prompt_arg: prompt})
135
prompt_ids, prompt_text = prompt_formats
136
137
is_tracing_enabled = await self.engine.is_tracing_enabled()
@@ -179,7 +177,8 @@ class OpenAIServingCompletion(OpenAIServing):
177
request_id,
178
created_time,
179
model_name,
182
- num_prompts=len(prompts))
180
+ num_prompts=len(prompts),
181
+ tokenizer=tokenizer)
182
183
# Non-streaming response
184
final_res_batch: List[Optional[RequestOutput]] = [None] * len(prompts)
@@ -191,7 +190,8 @@ class OpenAIServingCompletion(OpenAIServing):
190
return self.create_error_response("Client disconnected")
191
final_res_batch[i] = res
192
response = self.request_output_to_completion_response(
194
- final_res_batch, request, request_id, created_time, model_name)
193
+ final_res_batch, request, request_id, created_time, model_name,
194
+ tokenizer)
195
except ValueError as e:
196
# TODO: Use a vllm-specific Validation Error
197
return self.create_error_response(str(e))
@@ -218,6 +218,7 @@ class OpenAIServingCompletion(OpenAIServing):
218
created_time: int,
219
model_name: str,
220
num_prompts: int,
221
+ tokenizer: PreTrainedTokenizer,
222
) -> AsyncGenerator[str, None]:
223
assert request.n is not None
224
previous_texts = [""] * request.n * num_prompts
@@ -268,6 +269,7 @@ class OpenAIServingCompletion(OpenAIServing):
269
token_ids=delta_token_ids,
270
top_logprobs=out_logprobs,
271
num_output_top_logprobs=request.logprobs,
272
+ tokenizer=tokenizer,
273
initial_text_offset=len(previous_texts[i]),
274
)
275
else:
@@ -336,6 +338,7 @@ class OpenAIServingCompletion(OpenAIServing):
338
request_id: str,
339
created_time: int,
340
model_name: str,
341
+ tokenizer: PreTrainedTokenizer,
342
) -> CompletionResponse:
343
choices: List[CompletionResponseChoice] = []
344
num_prompt_tokens = 0
@@ -367,6 +370,7 @@ class OpenAIServingCompletion(OpenAIServing):
370
logprobs = self._create_completion_logprobs(
371
token_ids=token_ids,
372
top_logprobs=out_logprobs,
373
+ tokenizer=tokenizer,
374
num_output_top_logprobs=request.logprobs,
375
)
376
else:
@@ -404,6 +408,7 @@ class OpenAIServingCompletion(OpenAIServing):
408
token_ids: GenericSequence[int],
409
top_logprobs: GenericSequence[Optional[Dict[int, Logprob]]],
410
num_output_top_logprobs: int,
411
+ tokenizer: PreTrainedTokenizer,
412
initial_text_offset: int = 0,
413
) -> CompletionLogProbs:
414
"""Create logprobs for OpenAI Completion API."""
@@ -417,13 +422,13 @@ class OpenAIServingCompletion(OpenAIServing):
422
for i, token_id in enumerate(token_ids):
423
step_top_logprobs = top_logprobs[i]
424
if step_top_logprobs is None:
420
- token = self.tokenizer.decode(token_id)
425
+ token = tokenizer.decode(token_id)
426
out_tokens.append(token)
427
out_token_logprobs.append(None)
428
out_top_logprobs.append(None)
429
else:
430
token = self._get_decoded_token(step_top_logprobs[token_id],
426
- token_id)
431
+ token_id, tokenizer)
432
token_logprob = max(step_top_logprobs[token_id].logprob,
433
-9999.0)
434
out_tokens.append(token)
@@ -436,7 +441,7 @@ class OpenAIServingCompletion(OpenAIServing):
441
out_top_logprobs.append({
442
# Convert float("-inf") to the
443
# JSON-serializable float that OpenAI uses
439
- self._get_decoded_token(top_lp[1], top_lp[0]):
444
+ self._get_decoded_token(top_lp[1], top_lp[0], tokenizer):
445
max(top_lp[1].logprob, -9999.0)
446
for i, top_lp in enumerate(step_top_logprobs.items())
447
if num_output_top_logprobs >= i
vllm/entrypoints/openai/serving_embedding.py
+4
-7
@@ -89,14 +89,11 @@ class OpenAIServingEmbedding(OpenAIServing):
89
prompt_is_tokens, prompts = parse_prompt_format(request.input)
90
pooling_params = request.to_pooling_params()
91
92
+ tokenizer = await self.engine.get_tokenizer()
93
for i, prompt in enumerate(prompts):
93
- if prompt_is_tokens:
94
- prompt_formats = self._validate_prompt_and_tokenize(
95
- request, prompt_ids=prompt)
96
- else:
97
- prompt_formats = self._validate_prompt_and_tokenize(
98
- request, prompt=prompt)
99
-
94
+ prompt_arg = "prompt_ids" if prompt_is_tokens else "prompt"
95
+ prompt_formats = await self._validate_prompt_and_tokenize(
96
+ request, tokenizer, **{prompt_arg: prompt})
97
prompt_ids, prompt_text = prompt_formats
98
99
generator = self.engine.encode(
vllm/entrypoints/openai/serving_engine.py
+13
-17
@@ -5,6 +5,7 @@ from http import HTTPStatus
5
from typing import Any, Dict, List, Optional, Tuple, Union
6
7
from pydantic import Field
8
+from transformers import PreTrainedTokenizer
9
from typing_extensions import Annotated
10
11
from vllm.config import ModelConfig
@@ -19,7 +20,6 @@ from vllm.logger import init_logger
20
from vllm.lora.request import LoRARequest
21
from vllm.prompt_adapter.request import PromptAdapterRequest
22
from vllm.sequence import Logprob
22
-from vllm.transformers_utils.tokenizer import get_tokenizer
23
24
logger = init_logger(__name__)
25
@@ -52,14 +52,6 @@ class OpenAIServing:
52
self.model_config = model_config
53
self.max_model_len = model_config.max_model_len
54
55
- # A separate tokenizer to map token IDs to strings.
56
- self.tokenizer = get_tokenizer(
57
- model_config.tokenizer,
58
- tokenizer_mode=model_config.tokenizer_mode,
59
- tokenizer_revision=model_config.tokenizer_revision,
60
- trust_remote_code=model_config.trust_remote_code,
61
- truncation_side="left")
62
-
55
self.served_model_names = served_model_names
56
57
self.lora_requests = []
@@ -154,7 +146,8 @@ class OpenAIServing:
146
147
def _maybe_get_adapter(
148
self, request: Union[CompletionRequest, ChatCompletionRequest,
157
- EmbeddingRequest]
149
+ EmbeddingRequest, TokenizeRequest,
150
+ DetokenizeRequest]
151
) -> Tuple[Optional[str], Optional[Union[LoRARequest,
152
PromptAdapterRequest]]]:
153
if request.model in self.served_model_names:
@@ -168,11 +161,12 @@ class OpenAIServing:
161
# if _check_model has been called earlier, this will be unreachable
162
raise ValueError(f"The model `{request.model}` does not exist.")
163
171
- def _validate_prompt_and_tokenize(
164
+ async def _validate_prompt_and_tokenize(
165
self,
166
request: Union[ChatCompletionRequest, CompletionRequest,
167
DetokenizeRequest, EmbeddingRequest,
168
TokenizeRequest],
169
+ tokenizer: "PreTrainedTokenizer",
170
prompt: Optional[str] = None,
171
prompt_ids: Optional[List[int]] = None,
172
truncate_prompt_tokens: Optional[Annotated[int,
@@ -181,7 +175,7 @@ class OpenAIServing:
175
) -> Tuple[List[int], str]:
176
if not (prompt or prompt_ids):
177
raise ValueError("Either prompt or prompt_ids should be provided.")
184
- if (prompt and prompt_ids):
178
+ if prompt and prompt_ids:
179
raise ValueError(
180
"Only one of prompt or prompt_ids should be provided.")
181
@@ -200,14 +194,14 @@ class OpenAIServing:
194
"truncation": True,
195
"max_length": truncate_prompt_tokens,
196
})
203
- input_ids = self.tokenizer(prompt, **tokenizer_kwargs).input_ids
197
+ input_ids = tokenizer(prompt, **tokenizer_kwargs).input_ids
198
elif truncate_prompt_tokens is not None:
199
input_ids = prompt_ids[-truncate_prompt_tokens:]
200
else:
201
input_ids = prompt_ids
202
209
- input_text = prompt if prompt is not None else self.tokenizer.decode(
210
- prompt_ids)
203
+ input_text = prompt if prompt is not None else tokenizer.decode(
204
+ input_ids)
205
token_num = len(input_ids)
206
207
# Note: EmbeddingRequest doesn't have max_tokens
@@ -245,7 +239,9 @@ class OpenAIServing:
239
else:
240
return input_ids, input_text
241
248
- def _get_decoded_token(self, logprob: Logprob, token_id: int) -> str:
242
+ @staticmethod
243
+ def _get_decoded_token(logprob: Logprob, token_id: int,
244
+ tokenizer: PreTrainedTokenizer) -> str:
245
if logprob.decoded_token is not None:
246
return logprob.decoded_token
251
- return self.tokenizer.decode(token_id)
247
+ return tokenizer.decode(token_id)
vllm/entrypoints/openai/serving_tokenization.py
+20
-10
@@ -9,7 +9,8 @@ from vllm.entrypoints.openai.protocol import (DetokenizeRequest,
9
DetokenizeResponse,
10
TokenizeRequest,
11
TokenizeResponse)
12
-from vllm.entrypoints.openai.serving_engine import OpenAIServing
12
+from vllm.entrypoints.openai.serving_engine import (LoRAModulePath,
13
+ OpenAIServing)
14
15
16
class OpenAIServingTokenization(OpenAIServing):
@@ -18,13 +19,15 @@ class OpenAIServingTokenization(OpenAIServing):
19
engine: AsyncLLMEngine,
20
model_config: ModelConfig,
21
served_model_names: List[str],
22
+ lora_modules: Optional[List[LoRAModulePath]] = None,
23
chat_template: Optional[str] = None):
24
super().__init__(engine=engine,
25
model_config=model_config,
26
served_model_names=served_model_names,
25
- lora_modules=None)
27
+ lora_modules=lora_modules)
28
27
- load_chat_template(self, chat_template)
29
+ # If this is None we use the tokenizer's default chat template
30
+ self.chat_template = load_chat_template(chat_template)
31
32
async def create_tokenize(self,
33
request: TokenizeRequest) -> TokenizeResponse:
@@ -40,20 +43,25 @@ class OpenAIServingTokenization(OpenAIServing):
43
return self.create_error_response(
44
"Only one of `prompt` or `messages` should be provided.")
45
46
+ _, lora_request = self._maybe_get_adapter(request)
47
+ tokenizer = await self.engine.get_tokenizer(lora_request)
48
if request.messages:
49
conversation: List[ConversationMessage] = []
50
51
for message in request.messages:
47
- conversation.extend(
48
- parse_chat_message_content(self, message).messages)
52
+ result = parse_chat_message_content(message, self.model_config,
53
+ tokenizer)
54
+ conversation.extend(result.messages)
55
50
- request.prompt = self.tokenizer.apply_chat_template(
56
+ request.prompt = tokenizer.apply_chat_template(
57
add_generation_prompt=request.add_generation_prompt,
58
conversation=conversation,
53
- tokenize=False)
59
+ tokenize=False,
60
+ chat_template=self.chat_template)
61
55
- (input_ids, input_text) = self._validate_prompt_and_tokenize(
62
+ (input_ids, input_text) = await self._validate_prompt_and_tokenize(
63
request,
64
+ tokenizer,
65
prompt=request.prompt,
66
add_special_tokens=request.add_special_tokens)
67
@@ -67,7 +75,9 @@ class OpenAIServingTokenization(OpenAIServing):
75
if error_check_ret is not None:
76
return error_check_ret
77
70
- (input_ids, input_text) = self._validate_prompt_and_tokenize(
71
- request, prompt_ids=request.tokens)
78
+ _, lora_request = self._maybe_get_adapter(request)
79
+ tokenizer = await self.engine.get_tokenizer(lora_request)
80
+ (input_ids, input_text) = await self._validate_prompt_and_tokenize(
81
+ request, tokenizer, prompt_ids=request.tokens)
82
83
return DetokenizeResponse(prompt=input_text)
vllm/transformers_utils/detokenizer.py
+8
@@ -165,6 +165,12 @@ class Detokenizer:
165
return len(new_decoded_token_text)
166
167
168
+def _replace_none_with_empty(tokens: List[Optional[str]]):
169
+ for i, token in enumerate(tokens):
170
+ if token is None:
171
+ tokens[i] = ""
172
+
173
+
174
def _convert_tokens_to_string_with_added_encoders(
175
tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
176
output_tokens: List[str],
@@ -223,6 +229,8 @@ def convert_prompt_ids_to_tokens(
229
read_offset = len(new_tokens)
230
prefix_offset = max(
231
read_offset - INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET, 0)
232
+ # This is required to guard against out-of-vocab prompt token ids
233
+ _replace_none_with_empty(new_tokens)
234
return new_tokens, prefix_offset, read_offset
235
236
vllm/transformers_utils/tokenizer.py
+3
@@ -88,6 +88,9 @@ def get_tokenizer(
88
"Cannot use the fast tokenizer in slow tokenizer mode.")
89
kwargs["use_fast"] = False
90
91
+ if "truncation_side" not in kwargs:
92
+ kwargs["truncation_side"] = "left"
93
+
94
try:
95
tokenizer = AutoTokenizer.from_pretrained(
96
tokenizer_name,