A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Bugfix][Frontend] Raise exception when file-like chat template fails to be opened (#4292)
Cyrus Leung committed
Apr 24, 2024 at 02:19 UTC
1e8f4252aa163041094a8fedbb701a98d7087d7c
2 files changed
+29
-13
tests/async_engine/test_chat_template.py
+14
-5
@@ -76,20 +76,29 @@ def test_load_chat_template():
76
{% if add_generation_prompt and messages[-1]['role'] != 'assistant' %}{{ '<|im_start|>assistant\\n' }}{% endif %}""" # noqa: E501
77
78
79
-def test_no_load_chat_template():
79
+def test_no_load_chat_template_filelike():
80
# Testing chatml template
81
template = "../../examples/does_not_exist"
82
tokenizer = MockTokenizer()
83
84
+ mock_serving_chat = MockServingChat(tokenizer)
85
+
86
+ with pytest.raises(ValueError, match="looks like a file path"):
87
+ OpenAIServingChat._load_chat_template(mock_serving_chat,
88
+ chat_template=template)
89
+
90
+
91
+def test_no_load_chat_template_literallike():
92
+ # Testing chatml template
93
+ template = "{{ messages }}"
94
+ tokenizer = MockTokenizer()
95
+
96
mock_serving_chat = MockServingChat(tokenizer)
97
OpenAIServingChat._load_chat_template(mock_serving_chat,
98
chat_template=template)
99
template_content = tokenizer.chat_template
100
89
- # Test assertions
90
- assert template_content is not None
91
- # Hard coded value for template_chatml.jinja
92
- assert template_content == """../../examples/does_not_exist"""
101
+ assert template_content == template
102
103
104
@pytest.mark.asyncio
vllm/entrypoints/openai/serving_chat.py
+15
-8
@@ -319,23 +319,30 @@ class OpenAIServingChat(OpenAIServing):
319
return response
320
321
def _load_chat_template(self, chat_template):
322
+ tokenizer = self.tokenizer
323
+
324
if chat_template is not None:
325
try:
326
with open(chat_template, "r") as f:
325
- self.tokenizer.chat_template = f.read()
326
- except OSError:
327
+ tokenizer.chat_template = f.read()
328
+ except OSError as e:
329
+ JINJA_CHARS = "{}\n"
330
+ if not any(c in chat_template for c in JINJA_CHARS):
331
+ msg = (f"The supplied chat template ({chat_template}) "
332
+ f"looks like a file path, but it failed to be "
333
+ f"opened. Reason: {e}")
334
+ raise ValueError(msg) from e
335
+
336
# If opening a file fails, set chat template to be args to
337
# ensure we decode so our escape are interpreted correctly
329
- self.tokenizer.chat_template = codecs.decode(
338
+ tokenizer.chat_template = codecs.decode(
339
chat_template, "unicode_escape")
340
341
logger.info(
333
- f"Using supplied chat template:\n{self.tokenizer.chat_template}"
334
- )
335
- elif self.tokenizer.chat_template is not None:
342
+ f"Using supplied chat template:\n{tokenizer.chat_template}")
343
+ elif tokenizer.chat_template is not None:
344
logger.info(
337
- f"Using default chat template:\n{self.tokenizer.chat_template}"
338
- )
345
+ f"Using default chat template:\n{tokenizer.chat_template}")
346
else:
347
logger.warning(
348
"No chat template provided. Chat API will not work.")