@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cbcdf2c609

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 17:15 UTC

[Bugfix] Fix chat template loading (#15143)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Signed-off-by: Roger Wang <ywang@roblox.com> Co-authored-by: chaunceyjiang <chaunceyjiang@gmail.com> Co-authored-by: Roger Wang <ywang@roblox.com>

Cyrus Leung committed Mar 24, 2025 at 21:50 UTC cbcdf2c6091b162b94cbecf41312270e4b5d6ff2

7 files changed +187 -47

tests/entrypoints/openai/test_chat_template.py

@@ -107,8 +107,10 @@ def test_get_gen_prompt(model, template, add_generation_prompt,

107 # Call the function and get the result

108 result = apply_hf_chat_template(

109 tokenizer,

110 + trust_remote_code=True,

111 conversation=mock_request.messages,

112 chat_template=mock_request.chat_template or template_content,

113 + tools=None,

114 add_generation_prompt=mock_request.add_generation_prompt,

115 continue_final_message=mock_request.continue_final_message,

116 )

tests/entrypoints/openai/test_video.py

+2 -2

@@ -87,7 +87,7 @@ async def test_single_chat_session_video(client: openai.AsyncOpenAI,

87 choice = chat_completion.choices[0]

88 assert choice.finish_reason == "length"

89 assert chat_completion.usage == openai.types.CompletionUsage(

90 - completion_tokens=10, prompt_tokens=6299, total_tokens=6309)

90 + completion_tokens=10, prompt_tokens=6287, total_tokens=6297)

92 message = choice.message

93 message = chat_completion.choices[0].message

@@ -180,7 +180,7 @@ async def test_single_chat_session_video_base64encoded(

180 choice = chat_completion.choices[0]

181 assert choice.finish_reason == "length"

182 assert chat_completion.usage == openai.types.CompletionUsage(

183 - completion_tokens=10, prompt_tokens=6299, total_tokens=6309)

183 + completion_tokens=10, prompt_tokens=6287, total_tokens=6297)

184

185 message = choice.message

186 message = chat_completion.choices[0].message

tests/entrypoints/test_chat_utils.py

+62 -2

@@ -4,10 +4,13 @@ import warnings

4 from typing import Optional

6 import pytest

7 +from packaging.version import Version

8 +from transformers import __version__ as TRANSFORMERS_VERSION

10 from vllm.assets.image import ImageAsset

11 from vllm.config import ModelConfig

10 -from vllm.entrypoints.chat_utils import (_try_extract_ast, load_chat_template,

12 +from vllm.entrypoints.chat_utils import (_resolve_hf_chat_template,

13 + _try_extract_ast, load_chat_template,

14 parse_chat_messages,

15 parse_chat_messages_futures,

16 resolve_chat_template_content_format)

@@ -23,8 +26,10 @@ EXAMPLES_DIR = VLLM_PATH / "examples"

26 PHI3V_MODEL_ID = "microsoft/Phi-3.5-vision-instruct"

27 ULTRAVOX_MODEL_ID = "fixie-ai/ultravox-v0_5-llama-3_2-1b"

28 QWEN2VL_MODEL_ID = "Qwen/Qwen2-VL-2B-Instruct"

29 +QWEN25VL_MODEL_ID = "Qwen/Qwen2.5-VL-3B-Instruct"

30 MLLAMA_MODEL_ID = "meta-llama/Llama-3.2-11B-Vision-Instruct"

31 LLAMA_GUARD_MODEL_ID = "meta-llama/Llama-Guard-3-1B"

32 +HERMES_MODEL_ID = "NousResearch/Hermes-3-Llama-3.1-8B"

35 @pytest.fixture(scope="function")

@@ -703,25 +708,70 @@ def test_multimodal_image_parsing_matches_hf(model, image_url):

708

709 vllm_result = apply_hf_chat_template(

710 tokenizer,

711 + trust_remote_code=model_config.trust_remote_code,

712 conversation=conversation,

713 chat_template=None,

714 + tools=None,

715 add_generation_prompt=True,

716 )

717

718 assert hf_result == vllm_result

719

720

721 +@pytest.mark.parametrize(

722 + "model",

723 + [

724 + QWEN2VL_MODEL_ID, # tokenizer.chat_template is of type str

725 + HERMES_MODEL_ID, # tokenizer.chat_template is of type dict

726 + ])

727 +@pytest.mark.parametrize("use_tools", [True, False])

728 +def test_resolve_hf_chat_template(sample_json_schema, model, use_tools):

729 + """checks that chat_template is a dict type for HF models."""

730 +

731 + # Build the tokenizer group and grab the underlying tokenizer

732 + tokenizer_group = TokenizerGroup(

733 + model,

734 + enable_lora=False,

735 + max_num_seqs=5,

736 + max_input_length=None,

737 + )

738 + tokenizer = tokenizer_group.tokenizer

739 +

740 + tools = [{

741 + "type": "function",

742 + "function": {

743 + "name": "dummy_function_name",

744 + "description": "This is a dummy function",

745 + "parameters": sample_json_schema

746 + }

747 + }] if use_tools else None

748 +

749 + # Test detecting the tokenizer's chat_template

750 + chat_template = _resolve_hf_chat_template(

751 + tokenizer,

752 + chat_template=None,

753 + tools=tools,

754 + trust_remote_code=True,

755 + )

756 + assert isinstance(chat_template, str)

757 +

758 +

759 # yapf: disable

760 @pytest.mark.parametrize(

761 ("model", "expected_format"),

762 [(PHI3V_MODEL_ID, "string"),

763 (QWEN2VL_MODEL_ID, "openai"),

764 + (QWEN25VL_MODEL_ID, "openai"),

765 (ULTRAVOX_MODEL_ID, "string"),

766 (MLLAMA_MODEL_ID, "openai"),

767 (LLAMA_GUARD_MODEL_ID, "openai")],

768 )

769 # yapf: enable

770 def test_resolve_content_format_hf_defined(model, expected_format):

771 + if model == QWEN25VL_MODEL_ID and Version(TRANSFORMERS_VERSION) < Version(

772 + "4.49.0"):

773 + pytest.skip("Qwen2.5-VL requires transformers>=4.49.0")

774 +

775 tokenizer_group = TokenizerGroup(

776 model,

777 enable_lora=False,

@@ -730,7 +780,13 @@ def test_resolve_content_format_hf_defined(model, expected_format):

780 )

781 tokenizer = tokenizer_group.tokenizer

782

733 - chat_template = tokenizer.chat_template

783 + # Test detecting the tokenizer's chat_template

784 + chat_template = _resolve_hf_chat_template(

785 + tokenizer,

786 + chat_template=None,

787 + tools=None,

788 + trust_remote_code=True,

789 + )

790 assert isinstance(chat_template, str)

791

792 print("[TEXT]")

@@ -740,8 +796,10 @@ def test_resolve_content_format_hf_defined(model, expected_format):

796

797 resolved_format = resolve_chat_template_content_format(

798 None, # Test detecting the tokenizer's chat_template

799 + None,

800 "auto",

801 tokenizer,

802 + trust_remote_code=True,

803 )

804

805 assert resolved_format == expected_format

@@ -791,8 +849,10 @@ def test_resolve_content_format_examples(template_path, expected_format):

849

850 resolved_format = resolve_chat_template_content_format(

851 chat_template,

852 + None,

853 "auto",

854 dummy_tokenizer,

855 + trust_remote_code=True,

856 )

857

858 assert resolved_format == expected_format

tests/tool_use/utils.py

+4 -1

@@ -39,7 +39,10 @@ def ensure_system_prompt(messages: list[dict[str, Any]],

40 # universal args for all models go here. also good if you need to test locally

41 # and change type or KV cache quantization or something.

42 -ARGS: list[str] = ["--enable-auto-tool-choice", "--max-model-len", "1024"]

42 +ARGS: list[str] = [

43 + "--enable-auto-tool-choice", "--max-model-len", "1024", "--max-num-seqs",

44 + "256"

45 +]

47 CONFIGS: dict[str, ServerConfig] = {

48 "hermes": {

vllm/entrypoints/chat_utils.py

+106 -39

@@ -1,7 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

3 import asyncio

4 -import codecs

4 import json

5 from abc import ABC, abstractmethod

6 from collections import defaultdict, deque

@@ -30,7 +29,8 @@ from openai.types.chat.chat_completion_content_part_input_audio_param import (

29 InputAudio)

30 # yapf: enable

31 # pydantic needs the TypedDict from typing_extensions

33 -from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast

32 +from transformers import (PreTrainedTokenizer, PreTrainedTokenizerFast,

33 + ProcessorMixin)

34 from typing_extensions import Required, TypeAlias, TypedDict

36 from vllm.config import ModelConfig

@@ -306,24 +306,63 @@ def _detect_content_format(

306 return "openai"

307

308

309 +def _resolve_hf_chat_template(

310 + tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],

311 + chat_template: Optional[str],

312 + tools: Optional[list[dict[str, Any]]],

313 + *,

314 + trust_remote_code: bool,

315 +) -> Optional[str]:

316 + # 1st priority: The given chat template

317 + if chat_template is not None:

318 + return chat_template

319 +

320 + # 2nd priority: AutoProcessor chat template, unless tool calling is enabled

321 + if tools is None:

322 + try:

323 + processor = cached_get_processor(

324 + tokenizer.name_or_path,

325 + processor_cls=(PreTrainedTokenizer, PreTrainedTokenizerFast,

326 + ProcessorMixin),

327 + trust_remote_code=trust_remote_code,

328 + )

329 + if isinstance(processor, ProcessorMixin) and \

330 + processor.chat_template is not None:

331 + return processor.chat_template

332 + except Exception:

333 + logger.debug("Failed to load AutoProcessor chat template for %s",

334 + tokenizer.name_or_path, exc_info=True)

335 +

336 + # 3rd priority: AutoTokenizer chat template

337 + try:

338 + return tokenizer.get_chat_template(chat_template, tools=tools)

339 + except Exception:

340 + logger.debug("Failed to load AutoTokenizer chat template for %s",

341 + tokenizer.name_or_path, exc_info=True)

342 +

343 + return None

344 +

345 +

346 def _resolve_chat_template_content_format(

347 chat_template: Optional[str],

348 + tools: Optional[list[dict[str, Any]]],

349 given_format: ChatTemplateContentFormatOption,

350 tokenizer: AnyTokenizer,

351 + *,

352 + trust_remote_code: bool,

353 ) -> _ChatTemplateContentFormat:

354 if isinstance(tokenizer, (PreTrainedTokenizer, PreTrainedTokenizerFast)):

315 - tokenizer_chat_template = tokenizer.chat_template

316 - else:

317 - tokenizer_chat_template = None

318 -

319 - jinja_text: Optional[str]

320 - if isinstance(tokenizer_chat_template, str) and chat_template is None:

321 - jinja_text = tokenizer_chat_template

322 - elif (isinstance(tokenizer_chat_template, dict)

323 - and chat_template in tokenizer_chat_template):

324 - jinja_text = tokenizer_chat_template[chat_template]

355 + hf_chat_template = _resolve_hf_chat_template(

356 + tokenizer,

357 + chat_template=chat_template,

358 + trust_remote_code=trust_remote_code,

359 + tools=tools,

360 + )

361 else:

326 - jinja_text = load_chat_template(chat_template, is_literal=True)

362 + hf_chat_template = None

363 +

364 + jinja_text = (hf_chat_template if isinstance(hf_chat_template, str)

365 + else load_chat_template(chat_template, is_literal=True))

366

367 detected_format = ("string" if jinja_text is None else

368 _detect_content_format(jinja_text, default="string"))

@@ -332,17 +371,11 @@ def _resolve_chat_template_content_format(

371

372

373 @lru_cache

335 -def resolve_chat_template_content_format(

374 +def _log_chat_template_content_format(

375 chat_template: Optional[str],

376 given_format: ChatTemplateContentFormatOption,

338 - tokenizer: AnyTokenizer,

339 -) -> _ChatTemplateContentFormat:

340 - detected_format = _resolve_chat_template_content_format(

341 - chat_template,

342 - given_format,

343 - tokenizer,

344 - )

345 -

377 + detected_format: ChatTemplateContentFormatOption,

378 +):

379 logger.info(

380 "Detected the chat template content format to be '%s'. "

381 "You can set `--chat-template-content-format` to override this.",

@@ -360,6 +393,29 @@ def resolve_chat_template_content_format(

393 detected_format,

394 )

395

396 +

397 +def resolve_chat_template_content_format(

398 + chat_template: Optional[str],

399 + tools: Optional[list[dict[str, Any]]],

400 + given_format: ChatTemplateContentFormatOption,

401 + tokenizer: AnyTokenizer,

402 + *,

403 + trust_remote_code: bool = False,

404 +) -> _ChatTemplateContentFormat:

405 + detected_format = _resolve_chat_template_content_format(

406 + chat_template,

407 + tools,

408 + given_format,

409 + tokenizer,

410 + trust_remote_code=trust_remote_code,

411 + )

412 +

413 + _log_chat_template_content_format(

414 + chat_template,

415 + given_format=given_format,

416 + detected_format=detected_format,

417 + )

418 +

419 return detected_format

420

421

@@ -711,7 +767,7 @@ def validate_chat_template(chat_template: Optional[Union[Path, str]]):

767 f"{type(chat_template)} is not a valid chat template type")

768

769

714 -def load_chat_template(

770 +def _load_chat_template(

771 chat_template: Optional[Union[Path, str]],

772 *,

773 is_literal: bool = False,

@@ -724,7 +780,7 @@ def load_chat_template(

780 raise TypeError("chat_template is expected to be read directly "

781 "from its value")

782

727 - return codecs.decode(chat_template, "unicode_escape")

783 + return chat_template

784

785 try:

786 with open(chat_template) as f:

@@ -742,7 +798,18 @@ def load_chat_template(

798

799 # If opening a file fails, set chat template to be args to

800 # ensure we decode so our escape are interpreted correctly

745 - return load_chat_template(chat_template, is_literal=True)

801 + return _load_chat_template(chat_template, is_literal=True)

802 +

803 +

804 +_cached_load_chat_template = lru_cache(_load_chat_template)

805 +

806 +

807 +def load_chat_template(

808 + chat_template: Optional[Union[Path, str]],

809 + *,

810 + is_literal: bool = False,

811 +) -> Optional[str]:

812 + return _cached_load_chat_template(chat_template, is_literal=is_literal)

813

814

815 # TODO: Let user specify how to insert multimodal tokens into prompt

@@ -1067,23 +1134,20 @@ def apply_hf_chat_template(

1134 tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],

1135 conversation: list[ConversationMessage],

1136 chat_template: Optional[str],

1137 + tools: Optional[list[dict[str, Any]]],

1138 *,

1139 + trust_remote_code: bool = False,

1140 tokenize: bool = False, # Different from HF's default

1141 **kwargs: Any,

1142 ) -> str:

1074 - if chat_template is None:

1075 - chat_template = tokenizer.chat_template

1076 -

1077 - # FIXME: Temporary workaround for

1078 - # https://huggingface.co/mistral-community/pixtral-12b/discussions/31

1079 - if chat_template is None:

1080 - try:

1081 - processor = cached_get_processor(tokenizer.name_or_path)

1082 - chat_template = processor.chat_template

1083 - except Exception:

1084 - pass

1143 + hf_chat_template = _resolve_hf_chat_template(

1144 + tokenizer,

1145 + chat_template=chat_template,

1146 + tools=tools,

1147 + trust_remote_code=trust_remote_code,

1148 + )

1149

1086 - if chat_template is None:

1150 + if hf_chat_template is None:

1151 raise ValueError(

1152 "As of transformers v4.44, default chat template is no longer "

1153 "allowed, so you must provide a chat template if the tokenizer "

@@ -1091,7 +1155,8 @@ def apply_hf_chat_template(

1155

1156 return tokenizer.apply_chat_template(

1157 conversation=conversation, # type: ignore[arg-type]

1094 - chat_template=chat_template,

1158 + tools=tools, # type: ignore[arg-type]

1159 + chat_template=hf_chat_template,

1160 tokenize=tokenize,

1161 **kwargs,

1162 )

@@ -1100,7 +1165,8 @@ def apply_hf_chat_template(

1165 def apply_mistral_chat_template(

1166 tokenizer: MistralTokenizer,

1167 messages: list[ChatCompletionMessageParam],

1103 - chat_template: Optional[str] = None,

1168 + chat_template: Optional[str],

1169 + tools: Optional[list[dict[str, Any]]],

1170 **kwargs: Any,

1171 ) -> list[int]:

1172 if chat_template is not None:

@@ -1117,5 +1183,6 @@ def apply_mistral_chat_template(

1183

1184 return tokenizer.apply_chat_template(

1185 messages=messages,

1186 + tools=tools,

1187 **kwargs,

1188 )

vllm/entrypoints/llm.py

+5 -2

@@ -690,8 +690,10 @@ class LLM:

690 model_config = self.llm_engine.get_model_config()

691 resolved_content_format = resolve_chat_template_content_format(

692 chat_template,

693 + tools,

694 chat_template_content_format,

695 tokenizer,

696 + trust_remote_code=model_config.trust_remote_code,

697 )

698

699 prompts: list[Union[TokensPrompt, TextPrompt]] = []

@@ -713,18 +715,19 @@ class LLM:

715 tokenizer,

716 messages=msgs,

717 chat_template=chat_template,

718 + tools=tools,

719 add_generation_prompt=add_generation_prompt,

720 continue_final_message=continue_final_message,

718 - tools=tools,

721 )

722 else:

723 prompt_data = apply_hf_chat_template(

724 tokenizer,

725 + trust_remote_code=model_config.trust_remote_code,

726 conversation=conversation,

727 chat_template=chat_template,

728 + tools=tools,

729 add_generation_prompt=add_generation_prompt,

730 continue_final_message=continue_final_message,

727 - tools=tools,

731 )

732

733 prompt: Union[TokensPrompt, TextPrompt]

vllm/entrypoints/openai/serving_engine.py

+6 -1

@@ -379,14 +379,18 @@ class OpenAIServing:

379 add_special_tokens: bool = False,

380 ) -> tuple[list[ConversationMessage], Sequence[RequestPrompt],

381 list[TokensPrompt]]:

382 + model_config = self.model_config

383 +

384 resolved_content_format = resolve_chat_template_content_format(

385 chat_template,

386 + tool_dicts,

387 chat_template_content_format,

388 tokenizer,

389 + trust_remote_code=model_config.trust_remote_code,

390 )

391 conversation, mm_data_future = parse_chat_messages_futures(

392 messages,

389 - self.model_config,

393 + model_config,

394 tokenizer,

395 content_format=resolved_content_format,

396 )

@@ -410,6 +414,7 @@ class OpenAIServing:

414 else:

415 request_prompt = apply_hf_chat_template(

416 tokenizer,

417 + trust_remote_code=model_config.trust_remote_code,

418 conversation=conversation,

419 **_chat_template_kwargs,

420 )