@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 66d617e343
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Frontend] Gracefully handle missing chat template and fix CI failure (#7238)

Co-authored-by: Roger Wang <ywang@roblox.com>

Cyrus Leung committed Aug 7, 2024 at 17:12 UTC 66d617e3437e62a6650ffcb85b3190669d37a468
9 files changed +125 -69
tests/async_engine/test_chat_template.py
+8 -13
@@ -1,22 +1,16 @@
1 -import os
2 -import pathlib
3 -
1 import pytest
2
6 -from vllm.entrypoints.chat_utils import load_chat_template
3 +from vllm.entrypoints.chat_utils import apply_chat_template, load_chat_template
4 from vllm.entrypoints.openai.protocol import ChatCompletionRequest
5 from vllm.transformers_utils.tokenizer import get_tokenizer
6
10 -chatml_jinja_path = pathlib.Path(os.path.dirname(os.path.abspath(
11 - __file__))).parent.parent / "examples/template_chatml.jinja"
7 +from ..utils import VLLM_PATH
8 +
9 +chatml_jinja_path = VLLM_PATH / "examples/template_chatml.jinja"
10 assert chatml_jinja_path.exists()
11
12 # Define models, templates, and their corresponding expected outputs
13 MODEL_TEMPLATE_GENERATON_OUTPUT = [
16 - ("facebook/opt-125m", None, True,
17 - "Hello</s>Hi there!</s>What is the capital of</s>"),
18 - ("facebook/opt-125m", None, False,
19 - "Hello</s>Hi there!</s>What is the capital of</s>"),
14 ("facebook/opt-125m", chatml_jinja_path, True, """<|im_start|>user
15 Hello<|im_end|>
16 <|im_start|>assistant
@@ -93,11 +87,12 @@ def test_get_gen_prompt(model, template, add_generation_prompt,
87 add_generation_prompt=add_generation_prompt)
88
89 # Call the function and get the result
96 - result = tokenizer.apply_chat_template(
90 + result = apply_chat_template(
91 + tokenizer,
92 conversation=mock_request.messages,
98 - tokenize=False,
93 + chat_template=mock_request.chat_template or template_content,
94 add_generation_prompt=mock_request.add_generation_prompt,
100 - chat_template=mock_request.chat_template or template_content)
95 + )
96
97 # Test assertion
98 assert result == expected_output, (
tests/async_engine/test_openapi_server_ray.py
+7 -3
@@ -1,10 +1,12 @@
1 import openai # use the official client for correctness check
2 import pytest
3
4 -from ..utils import RemoteOpenAIServer
4 +from ..utils import VLLM_PATH, RemoteOpenAIServer
5
6 # any model with a chat template should work here
7 MODEL_NAME = "facebook/opt-125m"
8 +chatml_jinja_path = VLLM_PATH / "examples/template_chatml.jinja"
9 +assert chatml_jinja_path.exists()
10
11
12 @pytest.fixture(scope="module")
@@ -16,7 +18,9 @@ def server():
18 "--max-model-len",
19 "2048",
20 "--enforce-eager",
19 - "--engine-use-ray"
21 + "--engine-use-ray",
22 + "--chat-template",
23 + str(chatml_jinja_path),
24 ]
25
26 with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
@@ -83,7 +87,7 @@ async def test_single_chat_session(client: openai.AsyncOpenAI):
87 choice = chat_completion.choices[0]
88 assert choice.finish_reason == "length"
89 assert chat_completion.usage == openai.types.CompletionUsage(
86 - completion_tokens=10, prompt_tokens=13, total_tokens=23)
90 + completion_tokens=10, prompt_tokens=55, total_tokens=65)
91
92 message = choice.message
93 assert message.content is not None and len(message.content) >= 10
tests/entrypoints/openai/test_oot_registration.py
+55 -32
@@ -9,6 +9,11 @@ from vllm.model_executor.models.opt import OPTForCausalLM
9 from vllm.model_executor.sampling_metadata import SamplingMetadata
10 from vllm.utils import get_open_port
11
12 +from ...utils import VLLM_PATH, RemoteOpenAIServer
13 +
14 +chatml_jinja_path = VLLM_PATH / "examples/template_chatml.jinja"
15 +assert chatml_jinja_path.exists()
16 +
17
18 class MyOPTForCausalLM(OPTForCausalLM):
19
@@ -21,12 +26,25 @@ class MyOPTForCausalLM(OPTForCausalLM):
26 return logits
27
28
24 -def server_function(port):
29 +def server_function(port: int):
30 # register our dummy model
31 ModelRegistry.register_model("OPTForCausalLM", MyOPTForCausalLM)
27 - sys.argv = ["placeholder.py"] + \
28 - ("--model facebook/opt-125m --gpu-memory-utilization 0.10 "
29 - f"--dtype float32 --api-key token-abc123 --port {port}").split()
32 +
33 + sys.argv = ["placeholder.py"] + [
34 + "--model",
35 + "facebook/opt-125m",
36 + "--gpu-memory-utilization",
37 + "0.10",
38 + "--dtype",
39 + "float32",
40 + "--api-key",
41 + "token-abc123",
42 + "--port",
43 + str(port),
44 + "--chat-template",
45 + str(chatml_jinja_path),
46 + ]
47 +
48 import runpy
49 runpy.run_module('vllm.entrypoints.openai.api_server', run_name='__main__')
50
@@ -36,35 +54,40 @@ def test_oot_registration_for_api_server():
54 ctx = torch.multiprocessing.get_context()
55 server = ctx.Process(target=server_function, args=(port, ))
56 server.start()
39 - MAX_SERVER_START_WAIT_S = 60
40 - client = OpenAI(
41 - base_url=f"http://localhost:{port}/v1",
42 - api_key="token-abc123",
43 - )
44 - now = time.time()
45 - while True:
46 - try:
47 - completion = client.chat.completions.create(
48 - model="facebook/opt-125m",
49 - messages=[{
50 - "role": "system",
51 - "content": "You are a helpful assistant."
52 - }, {
53 - "role": "user",
54 - "content": "Hello!"
55 - }],
56 - temperature=0,
57 - )
58 - break
59 - except OpenAIError as e:
60 - if "Connection error" in str(e):
61 - time.sleep(3)
62 - if time.time() - now > MAX_SERVER_START_WAIT_S:
63 - raise RuntimeError("Server did not start in time") from e
64 - else:
65 - raise e
66 - server.kill()
57 +
58 + try:
59 + client = OpenAI(
60 + base_url=f"http://localhost:{port}/v1",
61 + api_key="token-abc123",
62 + )
63 + now = time.time()
64 + while True:
65 + try:
66 + completion = client.chat.completions.create(
67 + model="facebook/opt-125m",
68 + messages=[{
69 + "role": "system",
70 + "content": "You are a helpful assistant."
71 + }, {
72 + "role": "user",
73 + "content": "Hello!"
74 + }],
75 + temperature=0,
76 + )
77 + break
78 + except OpenAIError as e:
79 + if "Connection error" in str(e):
80 + time.sleep(3)
81 + if time.time() - now > RemoteOpenAIServer.MAX_START_WAIT_S:
82 + msg = "Server did not start in time"
83 + raise RuntimeError(msg) from e
84 + else:
85 + raise e
86 + finally:
87 + server.terminate()
88 +
89 generated_text = completion.choices[0].message.content
90 + assert generated_text is not None
91 # make sure only the first token is generated
92 rest = generated_text.replace("<s>", "")
93 assert rest == ""
tests/utils.py
+2 -2
@@ -50,7 +50,7 @@ VLLM_PATH = Path(__file__).parent.parent
50
51 class RemoteOpenAIServer:
52 DUMMY_API_KEY = "token-abc123" # vLLM's OpenAI server does not need API key
53 - MAX_SERVER_START_WAIT_S = 120 # wait for server to start for 120 seconds
53 + MAX_START_WAIT_S = 120 # wait for server to start for 120 seconds
54
55 def __init__(
56 self,
@@ -85,7 +85,7 @@ class RemoteOpenAIServer:
85 stdout=sys.stdout,
86 stderr=sys.stderr)
87 self._wait_for_server(url=self.url_for("health"),
88 - timeout=self.MAX_SERVER_START_WAIT_S)
88 + timeout=self.MAX_START_WAIT_S)
89
90 def __enter__(self):
91 return self
vllm/entrypoints/chat_utils.py
+34 -3
@@ -1,8 +1,9 @@
1 import codecs
2 from dataclasses import dataclass
3 from functools import lru_cache
4 -from typing import (Awaitable, Iterable, List, Optional, Tuple, Union, cast,
5 - final)
4 +from pathlib import Path
5 +from typing import (Any, Awaitable, Iterable, List, Optional, Tuple, Union,
6 + cast, final)
7
8 # yapf conflicts with isort for this block
9 # yapf: disable
@@ -22,6 +23,7 @@ from vllm.config import ModelConfig
23 from vllm.logger import init_logger
24 from vllm.multimodal import MultiModalDataDict
25 from vllm.multimodal.utils import async_get_and_parse_image
26 +from vllm.transformers_utils.tokenizer import AnyTokenizer
27
28 logger = init_logger(__name__)
29
@@ -69,13 +71,17 @@ class ChatMessageParseResult:
71 mm_futures: List[Awaitable[MultiModalDataDict]]
72
73
72 -def load_chat_template(chat_template: Optional[str]) -> Optional[str]:
74 +def load_chat_template(
75 + chat_template: Optional[Union[Path, str]]) -> Optional[str]:
76 if chat_template is None:
77 return None
78 try:
79 with open(chat_template, "r") as f:
80 resolved_chat_template = f.read()
81 except OSError as e:
82 + if isinstance(chat_template, Path):
83 + raise
84 +
85 JINJA_CHARS = "{}\n"
86 if not any(c in chat_template for c in JINJA_CHARS):
87 msg = (f"The supplied chat template ({chat_template}) "
@@ -208,3 +214,28 @@ def parse_chat_messages(
214 mm_futures.extend(parse_result.mm_futures)
215
216 return conversation, mm_futures
217 +
218 +
219 +def apply_chat_template(
220 + tokenizer: AnyTokenizer,
221 + conversation: List[ConversationMessage],
222 + chat_template: Optional[str],
223 + *,
224 + tokenize: bool = False, # Different from HF's default
225 + **kwargs: Any,
226 +) -> str:
227 + if chat_template is None and tokenizer.chat_template is None:
228 + raise ValueError(
229 + "As of transformers v4.44, default chat template is no longer "
230 + "allowed, so you must provide a chat template if the tokenizer "
231 + "does not define one.")
232 +
233 + prompt = tokenizer.apply_chat_template(
234 + conversation=conversation,
235 + chat_template=chat_template,
236 + tokenize=tokenize,
237 + **kwargs,
238 + )
239 + assert isinstance(prompt, str)
240 +
241 + return prompt
vllm/entrypoints/openai/protocol.py
+3 -2
@@ -190,8 +190,9 @@ class ChatCompletionRequest(OpenAIBaseModel):
190 default=None,
191 description=(
192 "A Jinja template to use for this conversion. "
193 - "If this is not passed, the model's default chat template will be "
194 - "used instead."),
193 + "As of transformers v4.44, default chat template is no longer "
194 + "allowed, so you must provide a chat template if the tokenizer "
195 + "does not define one."),
196 )
197 chat_template_kwargs: Optional[Dict[str, Any]] = Field(
198 default=None,
vllm/entrypoints/openai/serving_chat.py
+4 -4
@@ -10,6 +10,7 @@ from transformers import PreTrainedTokenizer
10 from vllm.config import ModelConfig
11 from vllm.engine.protocol import AsyncEngineClient
12 from vllm.entrypoints.chat_utils import (ConversationMessage,
13 + apply_chat_template,
14 load_chat_template,
15 parse_chat_messages)
16 from vllm.entrypoints.logger import RequestLogger
@@ -99,16 +100,15 @@ class OpenAIServingChat(OpenAIServing):
100 tool.model_dump() for tool in request.tools
101 ]
102
102 - prompt = tokenizer.apply_chat_template(
103 + prompt = apply_chat_template(
104 + tokenizer,
105 conversation=conversation,
104 - tokenize=False,
106 + chat_template=request.chat_template or self.chat_template,
107 add_generation_prompt=request.add_generation_prompt,
108 tools=tool_dicts,
109 documents=request.documents,
108 - chat_template=request.chat_template or self.chat_template,
110 **(request.chat_template_kwargs or {}),
111 )
111 - assert isinstance(prompt, str)
112 except Exception as e:
113 logger.error("Error in applying chat template from request: %s", e)
114 return self.create_error_response(str(e))
vllm/entrypoints/openai/serving_tokenization.py
+8 -6
@@ -2,7 +2,9 @@ from typing import List, Optional, Union
2
3 from vllm.config import ModelConfig
4 from vllm.engine.protocol import AsyncEngineClient
5 -from vllm.entrypoints.chat_utils import load_chat_template, parse_chat_messages
5 +from vllm.entrypoints.chat_utils import (apply_chat_template,
6 + load_chat_template,
7 + parse_chat_messages)
8 from vllm.entrypoints.logger import RequestLogger
9 # yapf conflicts with isort for this block
10 # yapf: disable
@@ -70,12 +72,12 @@ class OpenAIServingTokenization(OpenAIServing):
72 logger.warning(
73 "Multi-modal inputs are ignored during tokenization")
74
73 - prompt = tokenizer.apply_chat_template(
74 - add_generation_prompt=request.add_generation_prompt,
75 + prompt = apply_chat_template(
76 + tokenizer,
77 conversation=conversation,
76 - tokenize=False,
77 - chat_template=self.chat_template)
78 - assert isinstance(prompt, str)
78 + chat_template=self.chat_template,
79 + add_generation_prompt=request.add_generation_prompt,
80 + )
81 else:
82 prompt = request.prompt
83
vllm/transformers_utils/tokenizer.py
+4 -4
@@ -12,12 +12,12 @@ from vllm.lora.request import LoRARequest
12 from vllm.transformers_utils.tokenizers import BaichuanTokenizer
13 from vllm.utils import make_async
14
15 +from .tokenizer_group import AnyTokenizer
16 +
17 logger = init_logger(__name__)
18
19
18 -def get_cached_tokenizer(
19 - tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast]
20 -) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]:
20 +def get_cached_tokenizer(tokenizer: AnyTokenizer) -> AnyTokenizer:
21 """Get tokenizer with cached properties.
22
23 This will patch the tokenizer object in place.
@@ -63,7 +63,7 @@ def get_tokenizer(
63 revision: Optional[str] = None,
64 download_dir: Optional[str] = None,
65 **kwargs,
66 -) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]:
66 +) -> AnyTokenizer:
67 """Gets a tokenizer for the given model name via HuggingFace or ModelScope.
68 """
69 if VLLM_USE_MODELSCOPE: