@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1e8f4252aa
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Bugfix][Frontend] Raise exception when file-like chat template fails to be opened (#4292)

Cyrus Leung committed Apr 24, 2024 at 02:19 UTC 1e8f4252aa163041094a8fedbb701a98d7087d7c
2 files changed +29 -13
tests/async_engine/test_chat_template.py
+14 -5
@@ -76,20 +76,29 @@ def test_load_chat_template():
76 {% if add_generation_prompt and messages[-1]['role'] != 'assistant' %}{{ '<|im_start|>assistant\\n' }}{% endif %}""" # noqa: E501
77
78
79 -def test_no_load_chat_template():
79 +def test_no_load_chat_template_filelike():
80 # Testing chatml template
81 template = "../../examples/does_not_exist"
82 tokenizer = MockTokenizer()
83
84 + mock_serving_chat = MockServingChat(tokenizer)
85 +
86 + with pytest.raises(ValueError, match="looks like a file path"):
87 + OpenAIServingChat._load_chat_template(mock_serving_chat,
88 + chat_template=template)
89 +
90 +
91 +def test_no_load_chat_template_literallike():
92 + # Testing chatml template
93 + template = "{{ messages }}"
94 + tokenizer = MockTokenizer()
95 +
96 mock_serving_chat = MockServingChat(tokenizer)
97 OpenAIServingChat._load_chat_template(mock_serving_chat,
98 chat_template=template)
99 template_content = tokenizer.chat_template
100
89 - # Test assertions
90 - assert template_content is not None
91 - # Hard coded value for template_chatml.jinja
92 - assert template_content == """../../examples/does_not_exist"""
101 + assert template_content == template
102
103
104 @pytest.mark.asyncio
vllm/entrypoints/openai/serving_chat.py
+15 -8
@@ -319,23 +319,30 @@ class OpenAIServingChat(OpenAIServing):
319 return response
320
321 def _load_chat_template(self, chat_template):
322 + tokenizer = self.tokenizer
323 +
324 if chat_template is not None:
325 try:
326 with open(chat_template, "r") as f:
325 - self.tokenizer.chat_template = f.read()
326 - except OSError:
327 + tokenizer.chat_template = f.read()
328 + except OSError as e:
329 + JINJA_CHARS = "{}\n"
330 + if not any(c in chat_template for c in JINJA_CHARS):
331 + msg = (f"The supplied chat template ({chat_template}) "
332 + f"looks like a file path, but it failed to be "
333 + f"opened. Reason: {e}")
334 + raise ValueError(msg) from e
335 +
336 # If opening a file fails, set chat template to be args to
337 # ensure we decode so our escape are interpreted correctly
329 - self.tokenizer.chat_template = codecs.decode(
338 + tokenizer.chat_template = codecs.decode(
339 chat_template, "unicode_escape")
340
341 logger.info(
333 - f"Using supplied chat template:\n{self.tokenizer.chat_template}"
334 - )
335 - elif self.tokenizer.chat_template is not None:
342 + f"Using supplied chat template:\n{tokenizer.chat_template}")
343 + elif tokenizer.chat_template is not None:
344 logger.info(
337 - f"Using default chat template:\n{self.tokenizer.chat_template}"
338 - )
345 + f"Using default chat template:\n{tokenizer.chat_template}")
346 else:
347 logger.warning(
348 "No chat template provided. Chat API will not work.")