A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Frontend] Gracefully handle missing chat template and fix CI failure (#7238)
Co-authored-by: Roger Wang <ywang@roblox.com>
Cyrus Leung committed
Aug 7, 2024 at 17:12 UTC
66d617e3437e62a6650ffcb85b3190669d37a468
9 files changed
+125
-69
tests/async_engine/test_chat_template.py
+8
-13
@@ -1,22 +1,16 @@
1
-import os
2
-import pathlib
3
-
1
import pytest
2
6
-from vllm.entrypoints.chat_utils import load_chat_template
3
+from vllm.entrypoints.chat_utils import apply_chat_template, load_chat_template
4
from vllm.entrypoints.openai.protocol import ChatCompletionRequest
5
from vllm.transformers_utils.tokenizer import get_tokenizer
6
10
-chatml_jinja_path = pathlib.Path(os.path.dirname(os.path.abspath(
11
- __file__))).parent.parent / "examples/template_chatml.jinja"
7
+from ..utils import VLLM_PATH
8
+
9
+chatml_jinja_path = VLLM_PATH / "examples/template_chatml.jinja"
10
assert chatml_jinja_path.exists()
11
12
# Define models, templates, and their corresponding expected outputs
13
MODEL_TEMPLATE_GENERATON_OUTPUT = [
16
- ("facebook/opt-125m", None, True,
17
- "Hello</s>Hi there!</s>What is the capital of</s>"),
18
- ("facebook/opt-125m", None, False,
19
- "Hello</s>Hi there!</s>What is the capital of</s>"),
14
("facebook/opt-125m", chatml_jinja_path, True, """<|im_start|>user
15
Hello<|im_end|>
16
<|im_start|>assistant
@@ -93,11 +87,12 @@ def test_get_gen_prompt(model, template, add_generation_prompt,
87
add_generation_prompt=add_generation_prompt)
88
89
# Call the function and get the result
96
- result = tokenizer.apply_chat_template(
90
+ result = apply_chat_template(
91
+ tokenizer,
92
conversation=mock_request.messages,
98
- tokenize=False,
93
+ chat_template=mock_request.chat_template or template_content,
94
add_generation_prompt=mock_request.add_generation_prompt,
100
- chat_template=mock_request.chat_template or template_content)
95
+ )
96
97
# Test assertion
98
assert result == expected_output, (
tests/async_engine/test_openapi_server_ray.py
+7
-3
@@ -1,10 +1,12 @@
1
import openai # use the official client for correctness check
2
import pytest
3
4
-from ..utils import RemoteOpenAIServer
4
+from ..utils import VLLM_PATH, RemoteOpenAIServer
5
6
# any model with a chat template should work here
7
MODEL_NAME = "facebook/opt-125m"
8
+chatml_jinja_path = VLLM_PATH / "examples/template_chatml.jinja"
9
+assert chatml_jinja_path.exists()
10
11
12
@pytest.fixture(scope="module")
@@ -16,7 +18,9 @@ def server():
18
"--max-model-len",
19
"2048",
20
"--enforce-eager",
19
- "--engine-use-ray"
21
+ "--engine-use-ray",
22
+ "--chat-template",
23
+ str(chatml_jinja_path),
24
]
25
26
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
@@ -83,7 +87,7 @@ async def test_single_chat_session(client: openai.AsyncOpenAI):
87
choice = chat_completion.choices[0]
88
assert choice.finish_reason == "length"
89
assert chat_completion.usage == openai.types.CompletionUsage(
86
- completion_tokens=10, prompt_tokens=13, total_tokens=23)
90
+ completion_tokens=10, prompt_tokens=55, total_tokens=65)
91
92
message = choice.message
93
assert message.content is not None and len(message.content) >= 10
tests/entrypoints/openai/test_oot_registration.py
+55
-32
@@ -9,6 +9,11 @@ from vllm.model_executor.models.opt import OPTForCausalLM
9
from vllm.model_executor.sampling_metadata import SamplingMetadata
10
from vllm.utils import get_open_port
11
12
+from ...utils import VLLM_PATH, RemoteOpenAIServer
13
+
14
+chatml_jinja_path = VLLM_PATH / "examples/template_chatml.jinja"
15
+assert chatml_jinja_path.exists()
16
+
17
18
class MyOPTForCausalLM(OPTForCausalLM):
19
@@ -21,12 +26,25 @@ class MyOPTForCausalLM(OPTForCausalLM):
26
return logits
27
28
24
-def server_function(port):
29
+def server_function(port: int):
30
# register our dummy model
31
ModelRegistry.register_model("OPTForCausalLM", MyOPTForCausalLM)
27
- sys.argv = ["placeholder.py"] + \
28
- ("--model facebook/opt-125m --gpu-memory-utilization 0.10 "
29
- f"--dtype float32 --api-key token-abc123 --port {port}").split()
32
+
33
+ sys.argv = ["placeholder.py"] + [
34
+ "--model",
35
+ "facebook/opt-125m",
36
+ "--gpu-memory-utilization",
37
+ "0.10",
38
+ "--dtype",
39
+ "float32",
40
+ "--api-key",
41
+ "token-abc123",
42
+ "--port",
43
+ str(port),
44
+ "--chat-template",
45
+ str(chatml_jinja_path),
46
+ ]
47
+
48
import runpy
49
runpy.run_module('vllm.entrypoints.openai.api_server', run_name='__main__')
50
@@ -36,35 +54,40 @@ def test_oot_registration_for_api_server():
54
ctx = torch.multiprocessing.get_context()
55
server = ctx.Process(target=server_function, args=(port, ))
56
server.start()
39
- MAX_SERVER_START_WAIT_S = 60
40
- client = OpenAI(
41
- base_url=f"http://localhost:{port}/v1",
42
- api_key="token-abc123",
43
- )
44
- now = time.time()
45
- while True:
46
- try:
47
- completion = client.chat.completions.create(
48
- model="facebook/opt-125m",
49
- messages=[{
50
- "role": "system",
51
- "content": "You are a helpful assistant."
52
- }, {
53
- "role": "user",
54
- "content": "Hello!"
55
- }],
56
- temperature=0,
57
- )
58
- break
59
- except OpenAIError as e:
60
- if "Connection error" in str(e):
61
- time.sleep(3)
62
- if time.time() - now > MAX_SERVER_START_WAIT_S:
63
- raise RuntimeError("Server did not start in time") from e
64
- else:
65
- raise e
66
- server.kill()
57
+
58
+ try:
59
+ client = OpenAI(
60
+ base_url=f"http://localhost:{port}/v1",
61
+ api_key="token-abc123",
62
+ )
63
+ now = time.time()
64
+ while True:
65
+ try:
66
+ completion = client.chat.completions.create(
67
+ model="facebook/opt-125m",
68
+ messages=[{
69
+ "role": "system",
70
+ "content": "You are a helpful assistant."
71
+ }, {
72
+ "role": "user",
73
+ "content": "Hello!"
74
+ }],
75
+ temperature=0,
76
+ )
77
+ break
78
+ except OpenAIError as e:
79
+ if "Connection error" in str(e):
80
+ time.sleep(3)
81
+ if time.time() - now > RemoteOpenAIServer.MAX_START_WAIT_S:
82
+ msg = "Server did not start in time"
83
+ raise RuntimeError(msg) from e
84
+ else:
85
+ raise e
86
+ finally:
87
+ server.terminate()
88
+
89
generated_text = completion.choices[0].message.content
90
+ assert generated_text is not None
91
# make sure only the first token is generated
92
rest = generated_text.replace("<s>", "")
93
assert rest == ""
tests/utils.py
+2
-2
@@ -50,7 +50,7 @@ VLLM_PATH = Path(__file__).parent.parent
50
51
class RemoteOpenAIServer:
52
DUMMY_API_KEY = "token-abc123" # vLLM's OpenAI server does not need API key
53
- MAX_SERVER_START_WAIT_S = 120 # wait for server to start for 120 seconds
53
+ MAX_START_WAIT_S = 120 # wait for server to start for 120 seconds
54
55
def __init__(
56
self,
@@ -85,7 +85,7 @@ class RemoteOpenAIServer:
85
stdout=sys.stdout,
86
stderr=sys.stderr)
87
self._wait_for_server(url=self.url_for("health"),
88
- timeout=self.MAX_SERVER_START_WAIT_S)
88
+ timeout=self.MAX_START_WAIT_S)
89
90
def __enter__(self):
91
return self
vllm/entrypoints/chat_utils.py
+34
-3
@@ -1,8 +1,9 @@
1
import codecs
2
from dataclasses import dataclass
3
from functools import lru_cache
4
-from typing import (Awaitable, Iterable, List, Optional, Tuple, Union, cast,
5
- final)
4
+from pathlib import Path
5
+from typing import (Any, Awaitable, Iterable, List, Optional, Tuple, Union,
6
+ cast, final)
7
8
# yapf conflicts with isort for this block
9
# yapf: disable
@@ -22,6 +23,7 @@ from vllm.config import ModelConfig
23
from vllm.logger import init_logger
24
from vllm.multimodal import MultiModalDataDict
25
from vllm.multimodal.utils import async_get_and_parse_image
26
+from vllm.transformers_utils.tokenizer import AnyTokenizer
27
28
logger = init_logger(__name__)
29
@@ -69,13 +71,17 @@ class ChatMessageParseResult:
71
mm_futures: List[Awaitable[MultiModalDataDict]]
72
73
72
-def load_chat_template(chat_template: Optional[str]) -> Optional[str]:
74
+def load_chat_template(
75
+ chat_template: Optional[Union[Path, str]]) -> Optional[str]:
76
if chat_template is None:
77
return None
78
try:
79
with open(chat_template, "r") as f:
80
resolved_chat_template = f.read()
81
except OSError as e:
82
+ if isinstance(chat_template, Path):
83
+ raise
84
+
85
JINJA_CHARS = "{}\n"
86
if not any(c in chat_template for c in JINJA_CHARS):
87
msg = (f"The supplied chat template ({chat_template}) "
@@ -208,3 +214,28 @@ def parse_chat_messages(
214
mm_futures.extend(parse_result.mm_futures)
215
216
return conversation, mm_futures
217
+
218
+
219
+def apply_chat_template(
220
+ tokenizer: AnyTokenizer,
221
+ conversation: List[ConversationMessage],
222
+ chat_template: Optional[str],
223
+ *,
224
+ tokenize: bool = False, # Different from HF's default
225
+ **kwargs: Any,
226
+) -> str:
227
+ if chat_template is None and tokenizer.chat_template is None:
228
+ raise ValueError(
229
+ "As of transformers v4.44, default chat template is no longer "
230
+ "allowed, so you must provide a chat template if the tokenizer "
231
+ "does not define one.")
232
+
233
+ prompt = tokenizer.apply_chat_template(
234
+ conversation=conversation,
235
+ chat_template=chat_template,
236
+ tokenize=tokenize,
237
+ **kwargs,
238
+ )
239
+ assert isinstance(prompt, str)
240
+
241
+ return prompt
vllm/entrypoints/openai/protocol.py
+3
-2
@@ -190,8 +190,9 @@ class ChatCompletionRequest(OpenAIBaseModel):
190
default=None,
191
description=(
192
"A Jinja template to use for this conversion. "
193
- "If this is not passed, the model's default chat template will be "
194
- "used instead."),
193
+ "As of transformers v4.44, default chat template is no longer "
194
+ "allowed, so you must provide a chat template if the tokenizer "
195
+ "does not define one."),
196
)
197
chat_template_kwargs: Optional[Dict[str, Any]] = Field(
198
default=None,
vllm/entrypoints/openai/serving_chat.py
+4
-4
@@ -10,6 +10,7 @@ from transformers import PreTrainedTokenizer
10
from vllm.config import ModelConfig
11
from vllm.engine.protocol import AsyncEngineClient
12
from vllm.entrypoints.chat_utils import (ConversationMessage,
13
+ apply_chat_template,
14
load_chat_template,
15
parse_chat_messages)
16
from vllm.entrypoints.logger import RequestLogger
@@ -99,16 +100,15 @@ class OpenAIServingChat(OpenAIServing):
100
tool.model_dump() for tool in request.tools
101
]
102
102
- prompt = tokenizer.apply_chat_template(
103
+ prompt = apply_chat_template(
104
+ tokenizer,
105
conversation=conversation,
104
- tokenize=False,
106
+ chat_template=request.chat_template or self.chat_template,
107
add_generation_prompt=request.add_generation_prompt,
108
tools=tool_dicts,
109
documents=request.documents,
108
- chat_template=request.chat_template or self.chat_template,
110
**(request.chat_template_kwargs or {}),
111
)
111
- assert isinstance(prompt, str)
112
except Exception as e:
113
logger.error("Error in applying chat template from request: %s", e)
114
return self.create_error_response(str(e))
vllm/entrypoints/openai/serving_tokenization.py
+8
-6
@@ -2,7 +2,9 @@ from typing import List, Optional, Union
2
3
from vllm.config import ModelConfig
4
from vllm.engine.protocol import AsyncEngineClient
5
-from vllm.entrypoints.chat_utils import load_chat_template, parse_chat_messages
5
+from vllm.entrypoints.chat_utils import (apply_chat_template,
6
+ load_chat_template,
7
+ parse_chat_messages)
8
from vllm.entrypoints.logger import RequestLogger
9
# yapf conflicts with isort for this block
10
# yapf: disable
@@ -70,12 +72,12 @@ class OpenAIServingTokenization(OpenAIServing):
72
logger.warning(
73
"Multi-modal inputs are ignored during tokenization")
74
73
- prompt = tokenizer.apply_chat_template(
74
- add_generation_prompt=request.add_generation_prompt,
75
+ prompt = apply_chat_template(
76
+ tokenizer,
77
conversation=conversation,
76
- tokenize=False,
77
- chat_template=self.chat_template)
78
- assert isinstance(prompt, str)
78
+ chat_template=self.chat_template,
79
+ add_generation_prompt=request.add_generation_prompt,
80
+ )
81
else:
82
prompt = request.prompt
83
vllm/transformers_utils/tokenizer.py
+4
-4
@@ -12,12 +12,12 @@ from vllm.lora.request import LoRARequest
12
from vllm.transformers_utils.tokenizers import BaichuanTokenizer
13
from vllm.utils import make_async
14
15
+from .tokenizer_group import AnyTokenizer
16
+
17
logger = init_logger(__name__)
18
19
18
-def get_cached_tokenizer(
19
- tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast]
20
-) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]:
20
+def get_cached_tokenizer(tokenizer: AnyTokenizer) -> AnyTokenizer:
21
"""Get tokenizer with cached properties.
22
23
This will patch the tokenizer object in place.
@@ -63,7 +63,7 @@ def get_tokenizer(
63
revision: Optional[str] = None,
64
download_dir: Optional[str] = None,
65
**kwargs,
66
-) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]:
66
+) -> AnyTokenizer:
67
"""Gets a tokenizer for the given model name via HuggingFace or ModelScope.
68
"""
69
if VLLM_USE_MODELSCOPE: