@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e2fbaee725
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[BugFix][Frontend] Use LoRA tokenizer in OpenAI APIs (#6227)

Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>

Nick Hill committed Jul 18, 2024 at 00:13 UTC e2fbaee7258810bcd43725e9ca7f1444a88f91f3
16 files changed +267 -186
tests/async_engine/test_chat_template.py
+6 -27
@@ -1,6 +1,5 @@
1 import os
2 import pathlib
3 -from dataclasses import dataclass
3
4 import pytest
5
@@ -50,23 +49,9 @@ TEST_MESSAGES = [
49 ]
50
51
53 -@dataclass
54 -class MockTokenizer:
55 - chat_template = None
56 -
57 -
58 -@dataclass
59 -class MockServingChat:
60 - tokenizer: MockTokenizer
61 -
62 -
52 def test_load_chat_template():
53 # Testing chatml template
65 - tokenizer = MockTokenizer()
66 - mock_serving_chat = MockServingChat(tokenizer)
67 - load_chat_template(mock_serving_chat, chat_template=chatml_jinja_path)
68 -
69 - template_content = tokenizer.chat_template
54 + template_content = load_chat_template(chat_template=chatml_jinja_path)
55
56 # Test assertions
57 assert template_content is not None
@@ -78,22 +63,16 @@ def test_load_chat_template():
63 def test_no_load_chat_template_filelike():
64 # Testing chatml template
65 template = "../../examples/does_not_exist"
81 - tokenizer = MockTokenizer()
82 -
83 - mock_serving_chat = MockServingChat(tokenizer)
66
67 with pytest.raises(ValueError, match="looks like a file path"):
86 - load_chat_template(mock_serving_chat, chat_template=template)
68 + load_chat_template(chat_template=template)
69
70
71 def test_no_load_chat_template_literallike():
72 # Testing chatml template
73 template = "{{ messages }}"
92 - tokenizer = MockTokenizer()
74
94 - mock_serving_chat = MockServingChat(tokenizer)
95 - load_chat_template(mock_serving_chat, chat_template=template)
96 - template_content = tokenizer.chat_template
75 + template_content = load_chat_template(chat_template=template)
76
77 assert template_content == template
78
@@ -105,8 +84,7 @@ def test_get_gen_prompt(model, template, add_generation_prompt,
84 expected_output):
85 # Initialize the tokenizer
86 tokenizer = get_tokenizer(tokenizer_name=model)
108 - mock_serving_chat = MockServingChat(tokenizer)
109 - load_chat_template(mock_serving_chat, chat_template=template)
87 + template_content = load_chat_template(chat_template=template)
88
89 # Create a mock request object using keyword arguments
90 mock_request = ChatCompletionRequest(
@@ -118,7 +96,8 @@ def test_get_gen_prompt(model, template, add_generation_prompt,
96 result = tokenizer.apply_chat_template(
97 conversation=mock_request.messages,
98 tokenize=False,
121 - add_generation_prompt=mock_request.add_generation_prompt)
99 + add_generation_prompt=mock_request.add_generation_prompt,
100 + chat_template=mock_request.chat_template or template_content)
101
102 # Test assertion
103 assert result == expected_output, (
tests/entrypoints/openai/test_chat.py
+4 -9
@@ -7,11 +7,11 @@ import jsonschema
7 import openai # use the official client for correctness check
8 import pytest
9 import torch
10 -# downloading lora to test lora requests
11 -from huggingface_hub import snapshot_download
10 from openai import BadRequestError
11
12 from ...utils import RemoteOpenAIServer
13 +from .test_completion import zephyr_lora_added_tokens_files # noqa: F401
14 +from .test_completion import zephyr_lora_files # noqa: F401
15
16 # any model with a chat template should work here
17 MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
@@ -21,12 +21,7 @@ LORA_NAME = "typeof/zephyr-7b-beta-lora"
21
22
23 @pytest.fixture(scope="module")
24 -def zephyr_lora_files():
25 - return snapshot_download(repo_id=LORA_NAME)
26 -
27 -
28 -@pytest.fixture(scope="module")
29 -def server(zephyr_lora_files):
24 +def server(zephyr_lora_files, zephyr_lora_added_tokens_files): # noqa: F811
25 args = [
26 # use half precision for speed and memory savings in CI environment
27 "--dtype",
@@ -38,7 +33,7 @@ def server(zephyr_lora_files):
33 "--enable-lora",
34 "--lora-modules",
35 f"zephyr-lora={zephyr_lora_files}",
41 - f"zephyr-lora2={zephyr_lora_files}",
36 + f"zephyr-lora2={zephyr_lora_added_tokens_files}",
37 "--max-lora-rank",
38 "64",
39 "--max-cpu-loras",
tests/entrypoints/openai/test_completion.py
+49 -2
@@ -1,6 +1,8 @@
1 # imports for guided decoding tests
2 import json
3 import re
4 +import shutil
5 +from tempfile import TemporaryDirectory
6 from typing import List
7
8 import jsonschema
@@ -9,6 +11,7 @@ import pytest
11 # downloading lora to test lora requests
12 from huggingface_hub import snapshot_download
13 from openai import BadRequestError
14 +from transformers import AutoTokenizer
15
16 from vllm.transformers_utils.tokenizer import get_tokenizer
17
@@ -30,13 +33,29 @@ def zephyr_lora_files():
33 return snapshot_download(repo_id=LORA_NAME)
34
35
36 +@pytest.fixture(scope="module")
37 +def zephyr_lora_added_tokens_files(zephyr_lora_files):
38 + tmp_dir = TemporaryDirectory()
39 + tmp_model_dir = f"{tmp_dir.name}/zephyr"
40 + shutil.copytree(zephyr_lora_files, tmp_model_dir)
41 + tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
42 + # Copy tokenizer to adapter and add some unique tokens
43 + # 32000, 32001, 32002
44 + added = tokenizer.add_tokens(["vllm1", "vllm2", "vllm3"],
45 + special_tokens=True)
46 + assert added == 3
47 + tokenizer.save_pretrained(tmp_model_dir)
48 + yield tmp_model_dir
49 + tmp_dir.cleanup()
50 +
51 +
52 @pytest.fixture(scope="module")
53 def zephyr_pa_files():
54 return snapshot_download(repo_id=PA_NAME)
55
56
57 @pytest.fixture(scope="module")
39 -def server(zephyr_lora_files, zephyr_pa_files):
58 +def server(zephyr_lora_files, zephyr_lora_added_tokens_files, zephyr_pa_files):
59 args = [
60 # use half precision for speed and memory savings in CI environment
61 "--dtype",
@@ -50,7 +69,7 @@ def server(zephyr_lora_files, zephyr_pa_files):
69 "--enable-lora",
70 "--lora-modules",
71 f"zephyr-lora={zephyr_lora_files}",
53 - f"zephyr-lora2={zephyr_lora_files}",
72 + f"zephyr-lora2={zephyr_lora_added_tokens_files}",
73 "--max-lora-rank",
74 "64",
75 "--max-cpu-loras",
@@ -111,6 +130,34 @@ async def test_single_completion(client: openai.AsyncOpenAI, model_name: str,
130 assert len(completion.choices[0].text) >= 1
131
132
133 +@pytest.mark.asyncio
134 +async def test_added_lora_tokens(client: openai.AsyncOpenAI):
135 + # test using token IDs
136 + completion = await client.completions.create(
137 + model="zephyr-lora2",
138 + prompt=[0, 0, 32000, 32001, 32002],
139 + echo=True,
140 + max_tokens=5,
141 + temperature=0.0,
142 + )
143 + # Added tokens should appear in tokenized prompt
144 + assert completion.choices[0].text.startswith("<unk><unk>vllm1vllm2vllm3")
145 +
146 +
147 +@pytest.mark.asyncio
148 +async def test_added_lora_tokens_base_model(client: openai.AsyncOpenAI):
149 + # test using token IDs
150 + completion = await client.completions.create(
151 + model=MODEL_NAME,
152 + prompt=[0, 0, 32000, 32001, 32002],
153 + echo=True,
154 + max_tokens=5,
155 + temperature=0.0,
156 + )
157 + # Added tokens should not appear in tokenized prompt
158 + assert "vllm" not in completion.choices[0].text
159 +
160 +
161 @pytest.mark.asyncio
162 @pytest.mark.parametrize(
163 # first test base model, then test loras, then test prompt adapters
tests/entrypoints/openai/test_serving_chat.py
+1 -2
@@ -38,5 +38,4 @@ async def _async_serving_chat_init():
38
39 def test_async_serving_chat_init():
40 serving_completion = asyncio.run(_async_serving_chat_init())
41 - assert serving_completion.tokenizer is not None
42 - assert serving_completion.tokenizer.chat_template == CHAT_TEMPLATE
41 + assert serving_completion.chat_template == CHAT_TEMPLATE
tests/entrypoints/openai/test_tokenization.py
+40 -16
@@ -5,13 +5,15 @@ import requests
5 from vllm.transformers_utils.tokenizer import get_tokenizer
6
7 from ...utils import RemoteOpenAIServer
8 +from .test_completion import zephyr_lora_added_tokens_files # noqa: F401
9 +from .test_completion import zephyr_lora_files # noqa: F401
10
11 # any model with a chat template should work here
12 MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
13
14
15 @pytest.fixture(scope="module")
14 -def server():
16 +def server(zephyr_lora_added_tokens_files: str): # noqa: F811
17 args = [
18 # use half precision for speed and memory savings in CI environment
19 "--dtype",
@@ -21,12 +23,25 @@ def server():
23 "--enforce-eager",
24 "--max-num-seqs",
25 "128",
26 + # lora config
27 + "--enable-lora",
28 + "--lora-modules",
29 + f"zephyr-lora2={zephyr_lora_added_tokens_files}",
30 + "--max-lora-rank",
31 + "64",
32 ]
33
34 with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
35 yield remote_server
36
37
38 +@pytest.fixture(scope="module")
39 +def tokenizer_name(model_name: str,
40 + zephyr_lora_added_tokens_files: str): # noqa: F811
41 + return zephyr_lora_added_tokens_files if (
42 + model_name == "zephyr-lora2") else model_name
43 +
44 +
45 @pytest.fixture(scope="module")
46 def client(server):
47 return server.get_async_client()
@@ -34,16 +49,18 @@ def client(server):
49
50 @pytest.mark.asyncio
51 @pytest.mark.parametrize(
37 - "model_name",
38 - [MODEL_NAME],
52 + "model_name,tokenizer_name",
53 + [(MODEL_NAME, MODEL_NAME), ("zephyr-lora2", "zephyr-lora2")],
54 + indirect=["tokenizer_name"],
55 )
56 async def test_tokenize_completions(client: openai.AsyncOpenAI,
41 - model_name: str):
57 + model_name: str, tokenizer_name: str):
58 base_url = str(client.base_url)[:-3].strip("/")
43 - tokenizer = get_tokenizer(tokenizer_name=model_name, tokenizer_mode="fast")
59 + tokenizer = get_tokenizer(tokenizer_name=tokenizer_name,
60 + tokenizer_mode="fast")
61
62 for add_special in [False, True]:
46 - prompt = "This is a test prompt."
63 + prompt = "vllm1 This is a test prompt."
64 tokens = tokenizer.encode(prompt, add_special_tokens=add_special)
65
66 response = requests.post(base_url + "/tokenize",
@@ -63,12 +80,15 @@ async def test_tokenize_completions(client: openai.AsyncOpenAI,
80
81 @pytest.mark.asyncio
82 @pytest.mark.parametrize(
66 - "model_name",
67 - [MODEL_NAME],
83 + "model_name,tokenizer_name",
84 + [(MODEL_NAME, MODEL_NAME), ("zephyr-lora2", "zephyr-lora2")],
85 + indirect=["tokenizer_name"],
86 )
69 -async def test_tokenize_chat(client: openai.AsyncOpenAI, model_name: str):
87 +async def test_tokenize_chat(client: openai.AsyncOpenAI, model_name: str,
88 + tokenizer_name: str):
89 base_url = str(client.base_url)[:-3].strip("/")
71 - tokenizer = get_tokenizer(tokenizer_name=model_name, tokenizer_mode="fast")
90 + tokenizer = get_tokenizer(tokenizer_name=tokenizer_name,
91 + tokenizer_mode="fast")
92
93 for add_generation in [False, True]:
94 for add_special in [False, True]:
@@ -80,7 +100,7 @@ async def test_tokenize_chat(client: openai.AsyncOpenAI, model_name: str):
100 "content": "Nice to meet you!"
101 }, {
102 "role": "user",
83 - "content": "Can I ask a question?"
103 + "content": "Can I ask a question? vllm1"
104 }]
105
106 prompt = tokenizer.apply_chat_template(
@@ -108,16 +128,20 @@ async def test_tokenize_chat(client: openai.AsyncOpenAI, model_name: str):
128
129 @pytest.mark.asyncio
130 @pytest.mark.parametrize(
111 - "model_name",
112 - [MODEL_NAME],
131 + "model_name,tokenizer_name",
132 + [(MODEL_NAME, MODEL_NAME), ("zephyr-lora2", "zephyr-lora2")],
133 + indirect=["tokenizer_name"],
134 )
114 -async def test_detokenize(client: openai.AsyncOpenAI, model_name: str):
135 +async def test_detokenize(client: openai.AsyncOpenAI, model_name: str,
136 + tokenizer_name: str):
137 base_url = str(client.base_url)[:-3].strip("/")
116 - tokenizer = get_tokenizer(tokenizer_name=model_name, tokenizer_mode="fast")
138 + tokenizer = get_tokenizer(tokenizer_name=tokenizer_name,
139 + tokenizer_mode="fast")
140
118 - prompt = "This is a test prompt."
141 + prompt = "This is a test prompt. vllm1"
142 tokens = tokenizer.encode(prompt, add_special_tokens=False)
143
144 + print(f"CALLING {base_url} FOR {model_name}")
145 response = requests.post(base_url + "/detokenize",
146 json={
147 "model": model_name,
vllm/engine/async_llm_engine.py
+9 -4
@@ -480,11 +480,16 @@ class AsyncLLMEngine:
480 self.set_errored(exc)
481 self._request_tracker.propagate_exception(exc)
482
483 - async def get_tokenizer(self) -> "PreTrainedTokenizer":
483 + async def get_tokenizer(
484 + self,
485 + lora_request: Optional[LoRARequest] = None,
486 + ) -> "PreTrainedTokenizer":
487 if self.engine_use_ray:
485 - return await self.engine.get_tokenizer.remote() # type: ignore
486 - else:
487 - return self.engine.get_tokenizer()
488 + return await self.engine.get_tokenizer.remote( # type: ignore
489 + lora_request)
490 +
491 + return await (self.engine.get_tokenizer_group().
492 + get_lora_tokenizer_async(lora_request))
493
494 def start_background_loop(self) -> None:
495 """Start the background loop."""
vllm/engine/llm_engine.py
+5 -2
@@ -455,8 +455,11 @@ class LLMEngine:
455
456 return self.tokenizer
457
458 - def get_tokenizer(self) -> "PreTrainedTokenizer":
459 - return self.get_tokenizer_group().get_lora_tokenizer(None)
458 + def get_tokenizer(
459 + self,
460 + lora_request: Optional[LoRARequest] = None
461 + ) -> "PreTrainedTokenizer":
462 + return self.get_tokenizer_group().get_lora_tokenizer(lora_request)
463
464 def get_tokenizer_for_seq(self,
465 sequence: Sequence) -> "PreTrainedTokenizer":
vllm/entrypoints/openai/api_server.py
+2 -1
@@ -257,7 +257,8 @@ def run_server(args, llm_engine=None):
257 openai_serving_embedding = OpenAIServingEmbedding(engine, model_config,
258 served_model_names)
259 openai_serving_tokenization = OpenAIServingTokenization(
260 - engine, model_config, served_model_names, args.chat_template)
260 + engine, model_config, served_model_names, args.lora_modules,
261 + args.chat_template)
262 app.root_path = args.root_path
263
264 logger.info("Available routes are:")
vllm/entrypoints/openai/chat_utils.py
+33 -39
@@ -5,10 +5,11 @@ from typing import Awaitable, Iterable, List, Optional, TypedDict, cast, final
5
6 from openai.types.chat import (ChatCompletionContentPartImageParam,
7 ChatCompletionContentPartTextParam)
8 +from transformers import PreTrainedTokenizer
9
10 +from vllm.config import ModelConfig
11 from vllm.entrypoints.openai.protocol import (ChatCompletionContentPartParam,
12 ChatCompletionMessageParam)
11 -from vllm.entrypoints.openai.serving_engine import OpenAIServing
13 from vllm.logger import init_logger
14 from vllm.multimodal import MultiModalDataDict
15 from vllm.multimodal.utils import async_get_and_parse_image
@@ -29,40 +30,34 @@ class ChatMessageParseResult:
30 default_factory=list)
31
32
32 -def load_chat_template(engine: OpenAIServing, chat_template: Optional[str]):
33 - tokenizer = engine.tokenizer
34 -
35 - if chat_template is not None:
36 - try:
37 - with open(chat_template, "r") as f:
38 - tokenizer.chat_template = f.read()
39 - except OSError as e:
40 - JINJA_CHARS = "{}\n"
41 - if not any(c in chat_template for c in JINJA_CHARS):
42 - msg = (f"The supplied chat template ({chat_template}) "
43 - f"looks like a file path, but it failed to be "
44 - f"opened. Reason: {e}")
45 - raise ValueError(msg) from e
33 +def load_chat_template(chat_template: Optional[str]) -> Optional[str]:
34 + if chat_template is None:
35 + return None
36 + try:
37 + with open(chat_template, "r") as f:
38 + resolved_chat_template = f.read()
39 + except OSError as e:
40 + JINJA_CHARS = "{}\n"
41 + if not any(c in chat_template for c in JINJA_CHARS):
42 + msg = (f"The supplied chat template ({chat_template}) "
43 + f"looks like a file path, but it failed to be "
44 + f"opened. Reason: {e}")
45 + raise ValueError(msg) from e
46
47 - # If opening a file fails, set chat template to be args to
48 - # ensure we decode so our escape are interpreted correctly
49 - tokenizer.chat_template = codecs.decode(chat_template,
50 - "unicode_escape")
47 + # If opening a file fails, set chat template to be args to
48 + # ensure we decode so our escape are interpreted correctly
49 + resolved_chat_template = codecs.decode(chat_template, "unicode_escape")
50
52 - logger.info("Using supplied chat template:\n%s",
53 - tokenizer.chat_template)
54 - elif tokenizer.chat_template is not None:
55 - logger.info("Using default chat template:\n%s",
56 - tokenizer.chat_template)
57 - else:
58 - logger.warning("No chat template provided. Chat API will not work.")
51 + logger.info("Using supplied chat template:\n%s", resolved_chat_template)
52 + return resolved_chat_template
53
54
55 @lru_cache(maxsize=None)
62 -def _image_token_str(engine: OpenAIServing) -> Optional[str]:
56 +def _image_token_str(model_config: ModelConfig,
57 + tokenizer: PreTrainedTokenizer) -> Optional[str]:
58 # TODO: Let user specify how to insert image tokens into prompt
59 # (similar to chat template)
65 - model_type = engine.model_config.hf_config.model_type
60 + model_type = model_config.hf_config.model_type
61 if model_type == "phi3_v":
62 # Workaround since this token is not defined in the tokenizer
63 return "<|image_1|>"
@@ -70,17 +65,14 @@ def _image_token_str(engine: OpenAIServing) -> Optional[str]:
65 # These models do not use image tokens in the prompt
66 return None
67 if model_type.startswith("llava"):
73 - return engine.tokenizer.decode(
74 - engine.model_config.hf_config.image_token_index)
68 + return tokenizer.decode(model_config.hf_config.image_token_index)
69
76 - else:
77 - raise TypeError("Unknown model type: {model_type}")
70 + raise TypeError("Unknown model type: {model_type}")
71
72
73 # TODO: Let user specify how to insert image tokens into prompt
74 # (similar to chat template)
82 -def _get_full_image_text_prompt(engine: OpenAIServing, image_token_str: str,
83 - text_prompt: str) -> str:
75 +def _get_full_image_text_prompt(image_token_str: str, text_prompt: str) -> str:
76 """Combine image and text prompts for vision language model"""
77
78 # NOTE: For now we assume all model architectures use the same
@@ -89,9 +81,10 @@ def _get_full_image_text_prompt(engine: OpenAIServing, image_token_str: str,
81
82
83 def _parse_chat_message_content_parts(
92 - engine: OpenAIServing,
84 role: str,
85 parts: Iterable[ChatCompletionContentPartParam],
86 + model_config: ModelConfig,
87 + tokenizer: PreTrainedTokenizer,
88 ) -> ChatMessageParseResult:
89 texts: List[str] = []
90 mm_futures: List[Awaitable[MultiModalDataDict]] = []
@@ -122,7 +115,7 @@ def _parse_chat_message_content_parts(
115 text_prompt = "\n".join(texts)
116
117 if mm_futures:
125 - image_token_str = _image_token_str(engine)
118 + image_token_str = _image_token_str(model_config, tokenizer)
119 if image_token_str is not None:
120 if image_token_str in text_prompt:
121 logger.warning(
@@ -130,7 +123,6 @@ def _parse_chat_message_content_parts(
123 "Skipping prompt formatting.")
124 else:
125 text_prompt = _get_full_image_text_prompt(
133 - engine,
126 image_token_str=image_token_str,
127 text_prompt=text_prompt,
128 )
@@ -141,8 +133,9 @@ def _parse_chat_message_content_parts(
133
134
135 def parse_chat_message_content(
144 - engine: OpenAIServing,
136 message: ChatCompletionMessageParam,
137 + model_config: ModelConfig,
138 + tokenizer: PreTrainedTokenizer,
139 ) -> ChatMessageParseResult:
140 role = message["role"]
141 content = message.get("content")
@@ -153,4 +146,5 @@ def parse_chat_message_content(
146 messages = [ConversationMessage(role=role, content=content)]
147 return ChatMessageParseResult(messages=messages, mm_futures=[])
148
156 - return _parse_chat_message_content_parts(engine, role, content)
149 + return _parse_chat_message_content_parts(role, content, model_config,
150 + tokenizer)
vllm/entrypoints/openai/serving_chat.py
+44 -29
@@ -5,6 +5,7 @@ from typing import Sequence as GenericSequence
5 from typing import Union
6
7 from fastapi import Request
8 +from transformers import PreTrainedTokenizer
9
10 from vllm.config import ModelConfig
11 from vllm.engine.async_llm_engine import AsyncLLMEngine
@@ -49,7 +50,9 @@ class OpenAIServingChat(OpenAIServing):
50 lora_modules=lora_modules)
51
52 self.response_role = response_role
52 - load_chat_template(self, chat_template)
53 +
54 + # If this is None we use the tokenizer's default chat template
55 + self.chat_template = load_chat_template(chat_template)
56
57 async def create_chat_completion(
58 self,
@@ -71,11 +74,15 @@ class OpenAIServingChat(OpenAIServing):
74 return error_check_ret
75
76 try:
77 + _, lora_request = self._maybe_get_adapter(request)
78 + tokenizer = await self.engine.get_tokenizer(lora_request)
79 +
80 conversation: List[ConversationMessage] = []
81 mm_futures: List[Awaitable[MultiModalDataDict]] = []
82
83 for msg in request.messages:
78 - chat_parsed_result = parse_chat_message_content(self, msg)
84 + chat_parsed_result = parse_chat_message_content(
85 + msg, self.model_config, tokenizer)
86
87 conversation.extend(chat_parsed_result.messages)
88 mm_futures.extend(chat_parsed_result.mm_futures)
@@ -84,13 +91,13 @@ class OpenAIServingChat(OpenAIServing):
91 tool.model_dump() for tool in request.tools
92 ]
93
87 - prompt = self.tokenizer.apply_chat_template(
94 + prompt = tokenizer.apply_chat_template(
95 conversation=conversation,
96 tokenize=False,
97 add_generation_prompt=request.add_generation_prompt,
98 tools=tool_dicts,
99 documents=request.documents,
93 - chat_template=request.chat_template,
100 + chat_template=request.chat_template or self.chat_template,
101 **(request.chat_template_kwargs or {}),
102 )
103 except Exception as e:
@@ -112,19 +119,19 @@ class OpenAIServingChat(OpenAIServing):
119 request_id = f"cmpl-{random_uuid()}"
120 try:
121 # Tokenize/detokenize depending on prompt format (string/token list)
115 - prompt_ids, prompt_text = self._validate_prompt_and_tokenize(
122 + prompt_ids, prompt_text = await self._validate_prompt_and_tokenize(
123 request,
124 + tokenizer,
125 prompt=prompt,
126 add_special_tokens=request.add_special_tokens)
127 sampling_params = request.to_sampling_params()
120 - _, lora_request = self._maybe_get_adapter(request)
128 decoding_config = await self.engine.get_decoding_config()
129 guided_decoding_backend = request.guided_decoding_backend \
130 or decoding_config.guided_decoding_backend
131 guided_decode_logits_processor = (
125 - await get_guided_decoding_logits_processor(
126 - guided_decoding_backend, request, await
127 - self.engine.get_tokenizer()))
132 + await
133 + get_guided_decoding_logits_processor(guided_decoding_backend,
134 + request, tokenizer))
135 if guided_decode_logits_processor:
136 if sampling_params.logits_processors is None:
137 sampling_params.logits_processors = []
@@ -158,12 +165,12 @@ class OpenAIServingChat(OpenAIServing):
165 # Streaming response
166 if request.stream:
167 return self.chat_completion_stream_generator(
161 - request, result_generator, request_id, conversation)
168 + request, result_generator, request_id, conversation, tokenizer)
169 else:
170 try:
171 return await self.chat_completion_full_generator(
172 request, raw_request, result_generator, request_id,
166 - conversation)
173 + conversation, tokenizer)
174 except ValueError as e:
175 # TODO: Use a vllm-specific Validation Error
176 return self.create_error_response(str(e))
@@ -175,9 +182,12 @@ class OpenAIServingChat(OpenAIServing):
182 return request.messages[-1]["role"]
183
184 async def chat_completion_stream_generator(
178 - self, request: ChatCompletionRequest,
179 - result_generator: AsyncIterator[RequestOutput], request_id: str,
180 - conversation: List[ConversationMessage]
185 + self,
186 + request: ChatCompletionRequest,
187 + result_generator: AsyncIterator[RequestOutput],
188 + request_id: str,
189 + conversation: List[ConversationMessage],
190 + tokenizer: PreTrainedTokenizer,
191 ) -> AsyncGenerator[str, None]:
192 model_name = self.served_model_names[0]
193 created_time = int(time.time())
@@ -264,6 +274,7 @@ class OpenAIServingChat(OpenAIServing):
274 logprobs = self._create_chat_logprobs(
275 token_ids=delta_token_ids,
276 top_logprobs=out_logprobs,
277 + tokenizer=tokenizer,
278 num_output_top_logprobs=request.top_logprobs,
279 )
280 else:
@@ -352,9 +363,13 @@ class OpenAIServingChat(OpenAIServing):
363 yield "data: [DONE]\n\n"
364
365 async def chat_completion_full_generator(
355 - self, request: ChatCompletionRequest, raw_request: Optional[Request],
356 - result_generator: AsyncIterator[RequestOutput], request_id: str,
357 - conversation: List[ConversationMessage]
366 + self,
367 + request: ChatCompletionRequest,
368 + raw_request: Optional[Request],
369 + result_generator: AsyncIterator[RequestOutput],
370 + request_id: str,
371 + conversation: List[ConversationMessage],
372 + tokenizer: PreTrainedTokenizer,
373 ) -> Union[ErrorResponse, ChatCompletionResponse]:
374
375 model_name = self.served_model_names[0]
@@ -382,6 +397,7 @@ class OpenAIServingChat(OpenAIServing):
397 token_ids=token_ids,
398 top_logprobs=out_logprobs,
399 num_output_top_logprobs=request.top_logprobs,
400 + tokenizer=tokenizer,
401 )
402 else:
403 logprobs = None
@@ -436,16 +452,14 @@ class OpenAIServingChat(OpenAIServing):
452 return response
453
454 def _get_top_logprobs(
439 - self, logprobs: Dict[int, Logprob],
440 - top_logprobs: Optional[int]) -> List[ChatCompletionLogProb]:
455 + self, logprobs: Dict[int, Logprob], top_logprobs: Optional[int],
456 + tokenizer: PreTrainedTokenizer) -> List[ChatCompletionLogProb]:
457 return [
458 ChatCompletionLogProb(
443 - token=self._get_decoded_token(p[1], p[0]),
459 + token=(token := self._get_decoded_token(p[1], p[0],
460 + tokenizer)),
461 logprob=max(p[1].logprob, -9999.0),
445 - bytes=list(
446 - self._get_decoded_token(p[1],
447 - p[0]).encode("utf-8",
448 - errors="replace")))
462 + bytes=list(token.encode("utf-8", errors="replace")))
463 for i, p in enumerate(logprobs.items())
464 if top_logprobs and i < top_logprobs
465 ]
@@ -454,6 +468,7 @@ class OpenAIServingChat(OpenAIServing):
468 self,
469 token_ids: GenericSequence[int],
470 top_logprobs: GenericSequence[Optional[Dict[int, Logprob]]],
471 + tokenizer: PreTrainedTokenizer,
472 num_output_top_logprobs: Optional[int] = None,
473 ) -> ChatCompletionLogProbs:
474 """Create OpenAI-style logprobs."""
@@ -463,12 +478,11 @@ class OpenAIServingChat(OpenAIServing):
478 for i, token_id in enumerate(token_ids):
479 step_top_logprobs = top_logprobs[i]
480 if step_top_logprobs is None:
481 + token = tokenizer.decode(token_id)
482 logprobs_content.append(
483 ChatCompletionLogProbsContent(
468 - token=self.tokenizer.decode(token_id),
469 - bytes=list(
470 - self.tokenizer.decode(token_id).encode(
471 - "utf-8", errors="replace"))))
484 + token=token,
485 + bytes=list(token.encode("utf-8", errors="replace"))))
486 else:
487 logprobs_content.append(
488 ChatCompletionLogProbsContent(
@@ -479,6 +493,7 @@ class OpenAIServingChat(OpenAIServing):
493 step_top_logprobs[token_id].decoded_token.encode(
494 "utf-8", errors="replace")),
495 top_logprobs=self._get_top_logprobs(
482 - step_top_logprobs, num_output_top_logprobs)))
496 + step_top_logprobs, num_output_top_logprobs,
497 + tokenizer)))
498
499 return ChatCompletionLogProbs(content=logprobs_content)
vllm/entrypoints/openai/serving_completion.py
+26 -21
@@ -5,6 +5,7 @@ from typing import Sequence as GenericSequence
5 from typing import Tuple
6
7 from fastapi import Request
8 +from transformers import PreTrainedTokenizer
9
10 from vllm.config import ModelConfig
11 from vllm.engine.async_llm_engine import AsyncLLMEngine
@@ -100,20 +101,22 @@ class OpenAIServingCompletion(OpenAIServing):
101 # Schedule the request and get the result generator.
102 generators: List[AsyncIterator[RequestOutput]] = []
103 try:
103 - sampling_params = request.to_sampling_params()
104 adapter_type, adapter_request = self._maybe_get_adapter(request)
105 lora_request, prompt_adapter_request = None, None
106 if adapter_type == 'LoRA':
107 lora_request, prompt_adapter_request = adapter_request, None
108 elif adapter_type == 'PromptAdapter':
109 lora_request, prompt_adapter_request = None, adapter_request
110 + tokenizer = await self.engine.get_tokenizer(lora_request)
111 +
112 + sampling_params = request.to_sampling_params()
113 decoding_config = await self.engine.get_decoding_config()
114 guided_decoding_backend = request.guided_decoding_backend \
115 or decoding_config.guided_decoding_backend
116 guided_decode_logit_processor = (
114 - await get_guided_decoding_logits_processor(
115 - guided_decoding_backend, request, await
116 - self.engine.get_tokenizer()))
117 + await
118 + get_guided_decoding_logits_processor(guided_decoding_backend,
119 + request, tokenizer))
120 if guided_decode_logit_processor is not None:
121 if sampling_params.logits_processors is None:
122 sampling_params.logits_processors = []
@@ -122,18 +125,13 @@ class OpenAIServingCompletion(OpenAIServing):
125 prompt_is_tokens, prompts = parse_prompt_format(request.prompt)
126
127 for i, prompt in enumerate(prompts):
125 - if prompt_is_tokens:
126 - prompt_formats = self._validate_prompt_and_tokenize(
127 - request,
128 - prompt_ids=prompt,
129 - truncate_prompt_tokens=sampling_params.
130 - truncate_prompt_tokens)
131 - else:
132 - prompt_formats = self._validate_prompt_and_tokenize(
133 - request,
134 - prompt=prompt,
135 - truncate_prompt_tokens=sampling_params.
136 - truncate_prompt_tokens)
128 + prompt_arg = "prompt_ids" if prompt_is_tokens else "prompt"
129 + prompt_formats = await self._validate_prompt_and_tokenize(
130 + request,
131 + tokenizer,
132 + truncate_prompt_tokens=sampling_params.
133 + truncate_prompt_tokens,
134 + **{prompt_arg: prompt})
135 prompt_ids, prompt_text = prompt_formats
136
137 is_tracing_enabled = await self.engine.is_tracing_enabled()
@@ -179,7 +177,8 @@ class OpenAIServingCompletion(OpenAIServing):
177 request_id,
178 created_time,
179 model_name,
182 - num_prompts=len(prompts))
180 + num_prompts=len(prompts),
181 + tokenizer=tokenizer)
182
183 # Non-streaming response
184 final_res_batch: List[Optional[RequestOutput]] = [None] * len(prompts)
@@ -191,7 +190,8 @@ class OpenAIServingCompletion(OpenAIServing):
190 return self.create_error_response("Client disconnected")
191 final_res_batch[i] = res
192 response = self.request_output_to_completion_response(
194 - final_res_batch, request, request_id, created_time, model_name)
193 + final_res_batch, request, request_id, created_time, model_name,
194 + tokenizer)
195 except ValueError as e:
196 # TODO: Use a vllm-specific Validation Error
197 return self.create_error_response(str(e))
@@ -218,6 +218,7 @@ class OpenAIServingCompletion(OpenAIServing):
218 created_time: int,
219 model_name: str,
220 num_prompts: int,
221 + tokenizer: PreTrainedTokenizer,
222 ) -> AsyncGenerator[str, None]:
223 assert request.n is not None
224 previous_texts = [""] * request.n * num_prompts
@@ -268,6 +269,7 @@ class OpenAIServingCompletion(OpenAIServing):
269 token_ids=delta_token_ids,
270 top_logprobs=out_logprobs,
271 num_output_top_logprobs=request.logprobs,
272 + tokenizer=tokenizer,
273 initial_text_offset=len(previous_texts[i]),
274 )
275 else:
@@ -336,6 +338,7 @@ class OpenAIServingCompletion(OpenAIServing):
338 request_id: str,
339 created_time: int,
340 model_name: str,
341 + tokenizer: PreTrainedTokenizer,
342 ) -> CompletionResponse:
343 choices: List[CompletionResponseChoice] = []
344 num_prompt_tokens = 0
@@ -367,6 +370,7 @@ class OpenAIServingCompletion(OpenAIServing):
370 logprobs = self._create_completion_logprobs(
371 token_ids=token_ids,
372 top_logprobs=out_logprobs,
373 + tokenizer=tokenizer,
374 num_output_top_logprobs=request.logprobs,
375 )
376 else:
@@ -404,6 +408,7 @@ class OpenAIServingCompletion(OpenAIServing):
408 token_ids: GenericSequence[int],
409 top_logprobs: GenericSequence[Optional[Dict[int, Logprob]]],
410 num_output_top_logprobs: int,
411 + tokenizer: PreTrainedTokenizer,
412 initial_text_offset: int = 0,
413 ) -> CompletionLogProbs:
414 """Create logprobs for OpenAI Completion API."""
@@ -417,13 +422,13 @@ class OpenAIServingCompletion(OpenAIServing):
422 for i, token_id in enumerate(token_ids):
423 step_top_logprobs = top_logprobs[i]
424 if step_top_logprobs is None:
420 - token = self.tokenizer.decode(token_id)
425 + token = tokenizer.decode(token_id)
426 out_tokens.append(token)
427 out_token_logprobs.append(None)
428 out_top_logprobs.append(None)
429 else:
430 token = self._get_decoded_token(step_top_logprobs[token_id],
426 - token_id)
431 + token_id, tokenizer)
432 token_logprob = max(step_top_logprobs[token_id].logprob,
433 -9999.0)
434 out_tokens.append(token)
@@ -436,7 +441,7 @@ class OpenAIServingCompletion(OpenAIServing):
441 out_top_logprobs.append({
442 # Convert float("-inf") to the
443 # JSON-serializable float that OpenAI uses
439 - self._get_decoded_token(top_lp[1], top_lp[0]):
444 + self._get_decoded_token(top_lp[1], top_lp[0], tokenizer):
445 max(top_lp[1].logprob, -9999.0)
446 for i, top_lp in enumerate(step_top_logprobs.items())
447 if num_output_top_logprobs >= i
vllm/entrypoints/openai/serving_embedding.py
+4 -7
@@ -89,14 +89,11 @@ class OpenAIServingEmbedding(OpenAIServing):
89 prompt_is_tokens, prompts = parse_prompt_format(request.input)
90 pooling_params = request.to_pooling_params()
91
92 + tokenizer = await self.engine.get_tokenizer()
93 for i, prompt in enumerate(prompts):
93 - if prompt_is_tokens:
94 - prompt_formats = self._validate_prompt_and_tokenize(
95 - request, prompt_ids=prompt)
96 - else:
97 - prompt_formats = self._validate_prompt_and_tokenize(
98 - request, prompt=prompt)
99 -
94 + prompt_arg = "prompt_ids" if prompt_is_tokens else "prompt"
95 + prompt_formats = await self._validate_prompt_and_tokenize(
96 + request, tokenizer, **{prompt_arg: prompt})
97 prompt_ids, prompt_text = prompt_formats
98
99 generator = self.engine.encode(
vllm/entrypoints/openai/serving_engine.py
+13 -17
@@ -5,6 +5,7 @@ from http import HTTPStatus
5 from typing import Any, Dict, List, Optional, Tuple, Union
6
7 from pydantic import Field
8 +from transformers import PreTrainedTokenizer
9 from typing_extensions import Annotated
10
11 from vllm.config import ModelConfig
@@ -19,7 +20,6 @@ from vllm.logger import init_logger
20 from vllm.lora.request import LoRARequest
21 from vllm.prompt_adapter.request import PromptAdapterRequest
22 from vllm.sequence import Logprob
22 -from vllm.transformers_utils.tokenizer import get_tokenizer
23
24 logger = init_logger(__name__)
25
@@ -52,14 +52,6 @@ class OpenAIServing:
52 self.model_config = model_config
53 self.max_model_len = model_config.max_model_len
54
55 - # A separate tokenizer to map token IDs to strings.
56 - self.tokenizer = get_tokenizer(
57 - model_config.tokenizer,
58 - tokenizer_mode=model_config.tokenizer_mode,
59 - tokenizer_revision=model_config.tokenizer_revision,
60 - trust_remote_code=model_config.trust_remote_code,
61 - truncation_side="left")
62 -
55 self.served_model_names = served_model_names
56
57 self.lora_requests = []
@@ -154,7 +146,8 @@ class OpenAIServing:
146
147 def _maybe_get_adapter(
148 self, request: Union[CompletionRequest, ChatCompletionRequest,
157 - EmbeddingRequest]
149 + EmbeddingRequest, TokenizeRequest,
150 + DetokenizeRequest]
151 ) -> Tuple[Optional[str], Optional[Union[LoRARequest,
152 PromptAdapterRequest]]]:
153 if request.model in self.served_model_names:
@@ -168,11 +161,12 @@ class OpenAIServing:
161 # if _check_model has been called earlier, this will be unreachable
162 raise ValueError(f"The model `{request.model}` does not exist.")
163
171 - def _validate_prompt_and_tokenize(
164 + async def _validate_prompt_and_tokenize(
165 self,
166 request: Union[ChatCompletionRequest, CompletionRequest,
167 DetokenizeRequest, EmbeddingRequest,
168 TokenizeRequest],
169 + tokenizer: "PreTrainedTokenizer",
170 prompt: Optional[str] = None,
171 prompt_ids: Optional[List[int]] = None,
172 truncate_prompt_tokens: Optional[Annotated[int,
@@ -181,7 +175,7 @@ class OpenAIServing:
175 ) -> Tuple[List[int], str]:
176 if not (prompt or prompt_ids):
177 raise ValueError("Either prompt or prompt_ids should be provided.")
184 - if (prompt and prompt_ids):
178 + if prompt and prompt_ids:
179 raise ValueError(
180 "Only one of prompt or prompt_ids should be provided.")
181
@@ -200,14 +194,14 @@ class OpenAIServing:
194 "truncation": True,
195 "max_length": truncate_prompt_tokens,
196 })
203 - input_ids = self.tokenizer(prompt, **tokenizer_kwargs).input_ids
197 + input_ids = tokenizer(prompt, **tokenizer_kwargs).input_ids
198 elif truncate_prompt_tokens is not None:
199 input_ids = prompt_ids[-truncate_prompt_tokens:]
200 else:
201 input_ids = prompt_ids
202
209 - input_text = prompt if prompt is not None else self.tokenizer.decode(
210 - prompt_ids)
203 + input_text = prompt if prompt is not None else tokenizer.decode(
204 + input_ids)
205 token_num = len(input_ids)
206
207 # Note: EmbeddingRequest doesn't have max_tokens
@@ -245,7 +239,9 @@ class OpenAIServing:
239 else:
240 return input_ids, input_text
241
248 - def _get_decoded_token(self, logprob: Logprob, token_id: int) -> str:
242 + @staticmethod
243 + def _get_decoded_token(logprob: Logprob, token_id: int,
244 + tokenizer: PreTrainedTokenizer) -> str:
245 if logprob.decoded_token is not None:
246 return logprob.decoded_token
251 - return self.tokenizer.decode(token_id)
247 + return tokenizer.decode(token_id)
vllm/entrypoints/openai/serving_tokenization.py
+20 -10
@@ -9,7 +9,8 @@ from vllm.entrypoints.openai.protocol import (DetokenizeRequest,
9 DetokenizeResponse,
10 TokenizeRequest,
11 TokenizeResponse)
12 -from vllm.entrypoints.openai.serving_engine import OpenAIServing
12 +from vllm.entrypoints.openai.serving_engine import (LoRAModulePath,
13 + OpenAIServing)
14
15
16 class OpenAIServingTokenization(OpenAIServing):
@@ -18,13 +19,15 @@ class OpenAIServingTokenization(OpenAIServing):
19 engine: AsyncLLMEngine,
20 model_config: ModelConfig,
21 served_model_names: List[str],
22 + lora_modules: Optional[List[LoRAModulePath]] = None,
23 chat_template: Optional[str] = None):
24 super().__init__(engine=engine,
25 model_config=model_config,
26 served_model_names=served_model_names,
25 - lora_modules=None)
27 + lora_modules=lora_modules)
28
27 - load_chat_template(self, chat_template)
29 + # If this is None we use the tokenizer's default chat template
30 + self.chat_template = load_chat_template(chat_template)
31
32 async def create_tokenize(self,
33 request: TokenizeRequest) -> TokenizeResponse:
@@ -40,20 +43,25 @@ class OpenAIServingTokenization(OpenAIServing):
43 return self.create_error_response(
44 "Only one of `prompt` or `messages` should be provided.")
45
46 + _, lora_request = self._maybe_get_adapter(request)
47 + tokenizer = await self.engine.get_tokenizer(lora_request)
48 if request.messages:
49 conversation: List[ConversationMessage] = []
50
51 for message in request.messages:
47 - conversation.extend(
48 - parse_chat_message_content(self, message).messages)
52 + result = parse_chat_message_content(message, self.model_config,
53 + tokenizer)
54 + conversation.extend(result.messages)
55
50 - request.prompt = self.tokenizer.apply_chat_template(
56 + request.prompt = tokenizer.apply_chat_template(
57 add_generation_prompt=request.add_generation_prompt,
58 conversation=conversation,
53 - tokenize=False)
59 + tokenize=False,
60 + chat_template=self.chat_template)
61
55 - (input_ids, input_text) = self._validate_prompt_and_tokenize(
62 + (input_ids, input_text) = await self._validate_prompt_and_tokenize(
63 request,
64 + tokenizer,
65 prompt=request.prompt,
66 add_special_tokens=request.add_special_tokens)
67
@@ -67,7 +75,9 @@ class OpenAIServingTokenization(OpenAIServing):
75 if error_check_ret is not None:
76 return error_check_ret
77
70 - (input_ids, input_text) = self._validate_prompt_and_tokenize(
71 - request, prompt_ids=request.tokens)
78 + _, lora_request = self._maybe_get_adapter(request)
79 + tokenizer = await self.engine.get_tokenizer(lora_request)
80 + (input_ids, input_text) = await self._validate_prompt_and_tokenize(
81 + request, tokenizer, prompt_ids=request.tokens)
82
83 return DetokenizeResponse(prompt=input_text)
vllm/transformers_utils/detokenizer.py
+8
@@ -165,6 +165,12 @@ class Detokenizer:
165 return len(new_decoded_token_text)
166
167
168 +def _replace_none_with_empty(tokens: List[Optional[str]]):
169 + for i, token in enumerate(tokens):
170 + if token is None:
171 + tokens[i] = ""
172 +
173 +
174 def _convert_tokens_to_string_with_added_encoders(
175 tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
176 output_tokens: List[str],
@@ -223,6 +229,8 @@ def convert_prompt_ids_to_tokens(
229 read_offset = len(new_tokens)
230 prefix_offset = max(
231 read_offset - INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET, 0)
232 + # This is required to guard against out-of-vocab prompt token ids
233 + _replace_none_with_empty(new_tokens)
234 return new_tokens, prefix_offset, read_offset
235
236
vllm/transformers_utils/tokenizer.py
+3
@@ -88,6 +88,9 @@ def get_tokenizer(
88 "Cannot use the fast tokenizer in slow tokenizer mode.")
89 kwargs["use_fast"] = False
90
91 + if "truncation_side" not in kwargs:
92 + kwargs["truncation_side"] = "left"
93 +
94 try:
95 tokenizer = AutoTokenizer.from_pretrained(
96 tokenizer_name,