A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 11:00 UTC
[Frontend] Chat template fallbacks for multimodal models (#17805)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
May 8, 2025 at 14:05 UTC
96722aa81da3d103675a38cfe528dd719cd9fc0a
18 files changed
+219
-52
docs/source/serving/multimodal_inputs.md
+6
-3
@@ -213,10 +213,13 @@ Our OpenAI-compatible server accepts multi-modal data via the [Chat Completions
213
214
:::{important}
215
A chat template is **required** to use Chat Completions API.
216
+For HF format models, the default chat template is defined inside `chat_template.json` or `tokenizer_config.json`.
217
217
-Although most models come with a chat template, for others you have to define one yourself.
218
-The chat template can be inferred based on the documentation on the model's HuggingFace repo.
219
-For example, DeepSeek-VL2 requires a chat template that can be found here: <gh-file:examples/template_deepseek_vl2.jinja>
218
+If no default chat template is available, we will first look for a built-in fallback in <gh-file:vllm/transformers_utils/chat_templates/registry.py>.
219
+If no fallback is available, an error is raised and you have to provide the chat template manually via the `--chat-template` argument.
220
+
221
+For certain models, we provide alternative chat templates inside <gh-dir:vllm/examples>.
222
+For example, VLM2Vec uses <gh-file:examples/template_vlm2vec.jinja> which is different from the default one for Phi-3-Vision.
223
:::
224
225
### Image Inputs
examples/template_florence2.jinja
deleted
-3
@@ -1,3 +0,0 @@
1
-{%- for message in messages -%}
2
- {{- message['content'] -}}
3
-{%- endfor -%}
examples/template_paligemma.jinja
deleted
-3
@@ -1,3 +0,0 @@
1
-{%- for message in messages -%}
2
- {{- message['content'] -}}
3
-{%- endfor -%}
examples/template_qwen_vl.jinja
deleted
-3
@@ -1,3 +0,0 @@
1
-{%- for message in messages -%}
2
- {{- message['content'] -}}
3
-{%- endfor -%}
tests/entrypoints/openai/test_chat_template.py
+18
-2
@@ -2,11 +2,13 @@
2
3
import pytest
4
5
+from vllm.config import ModelConfig
6
from vllm.entrypoints.chat_utils import (apply_hf_chat_template,
7
load_chat_template)
8
from vllm.entrypoints.openai.protocol import ChatCompletionRequest
9
from vllm.transformers_utils.tokenizer import get_tokenizer
10
11
+from ...models.registry import HF_EXAMPLE_MODELS
12
from ...utils import VLLM_PATH
13
14
chatml_jinja_path = VLLM_PATH / "examples/template_chatml.jinja"
@@ -91,8 +93,22 @@ def test_no_load_chat_template_literallike():
93
MODEL_TEMPLATE_GENERATON_OUTPUT)
94
def test_get_gen_prompt(model, template, add_generation_prompt,
95
continue_final_message, expected_output):
96
+ model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
97
+ model_info.check_available_online(on_fail="skip")
98
+
99
+ model_config = ModelConfig(
100
+ model,
101
+ tokenizer=model_info.tokenizer or model,
102
+ tokenizer_mode=model_info.tokenizer_mode,
103
+ trust_remote_code=model_info.trust_remote_code,
104
+ hf_overrides=model_info.hf_overrides,
105
+ )
106
+
107
# Initialize the tokenizer
95
- tokenizer = get_tokenizer(tokenizer_name=model)
108
+ tokenizer = get_tokenizer(
109
+ tokenizer_name=model_config.tokenizer,
110
+ trust_remote_code=model_config.trust_remote_code,
111
+ )
112
template_content = load_chat_template(chat_template=template)
113
114
# Create a mock request object using keyword arguments
@@ -106,8 +122,8 @@ def test_get_gen_prompt(model, template, add_generation_prompt,
122
123
# Call the function and get the result
124
result = apply_hf_chat_template(
125
+ model_config,
126
tokenizer,
110
- trust_remote_code=True,
127
conversation=mock_request.messages,
128
chat_template=mock_request.chat_template or template_content,
129
tools=None,
tests/entrypoints/test_chat_utils.py
+96
-19
@@ -4,8 +4,6 @@ import warnings
4
from typing import Optional
5
6
import pytest
7
-from packaging.version import Version
8
-from transformers import __version__ as TRANSFORMERS_VERSION
7
8
from vllm.assets.image import ImageAsset
9
from vllm.config import ModelConfig
@@ -19,6 +17,7 @@ from vllm.multimodal import MultiModalDataDict
17
from vllm.multimodal.utils import encode_image_base64
18
from vllm.transformers_utils.tokenizer_group import TokenizerGroup
19
20
+from ..models.registry import HF_EXAMPLE_MODELS
21
from ..utils import VLLM_PATH
22
23
EXAMPLES_DIR = VLLM_PATH / "examples"
@@ -772,6 +771,7 @@ def test_multimodal_image_parsing_matches_hf(model, image_url):
771
enable_lora=False,
772
max_num_seqs=5,
773
max_input_length=None,
774
+ trust_remote_code=model_config.trust_remote_code,
775
)
776
tokenizer = tokenizer_group.tokenizer
777
@@ -793,8 +793,8 @@ def test_multimodal_image_parsing_matches_hf(model, image_url):
793
)
794
795
vllm_result = apply_hf_chat_template(
796
+ model_config,
797
tokenizer,
797
- trust_remote_code=model_config.trust_remote_code,
798
conversation=conversation,
799
chat_template=None,
800
tools=None,
@@ -813,6 +813,16 @@ def test_multimodal_image_parsing_matches_hf(model, image_url):
813
@pytest.mark.parametrize("use_tools", [True, False])
814
def test_resolve_hf_chat_template(sample_json_schema, model, use_tools):
815
"""checks that chat_template is a dict type for HF models."""
816
+ model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
817
+ model_info.check_available_online(on_fail="skip")
818
+
819
+ model_config = ModelConfig(
820
+ model,
821
+ tokenizer=model_info.tokenizer or model,
822
+ tokenizer_mode=model_info.tokenizer_mode,
823
+ trust_remote_code=model_info.trust_remote_code,
824
+ hf_overrides=model_info.hf_overrides,
825
+ )
826
827
# Build the tokenizer group and grab the underlying tokenizer
828
tokenizer_group = TokenizerGroup(
@@ -820,6 +830,7 @@ def test_resolve_hf_chat_template(sample_json_schema, model, use_tools):
830
enable_lora=False,
831
max_num_seqs=5,
832
max_input_length=None,
833
+ trust_remote_code=model_config.trust_remote_code,
834
)
835
tokenizer = tokenizer_group.tokenizer
836
@@ -834,10 +845,10 @@ def test_resolve_hf_chat_template(sample_json_schema, model, use_tools):
845
846
# Test detecting the tokenizer's chat_template
847
chat_template = resolve_hf_chat_template(
848
+ model_config,
849
tokenizer,
850
chat_template=None,
851
tools=tools,
840
- trust_remote_code=True,
852
)
853
assert isinstance(chat_template, str)
854
@@ -857,24 +868,32 @@ def test_resolve_hf_chat_template(sample_json_schema, model, use_tools):
868
)
869
# yapf: enable
870
def test_resolve_content_format_hf_defined(model, expected_format):
860
- if model == QWEN25VL_MODEL_ID and Version(TRANSFORMERS_VERSION) < Version(
861
- "4.49.0"):
862
- pytest.skip("Qwen2.5-VL requires transformers>=4.49.0")
871
+ model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
872
+ model_info.check_available_online(on_fail="skip")
873
+
874
+ model_config = ModelConfig(
875
+ model,
876
+ tokenizer=model_info.tokenizer or model,
877
+ tokenizer_mode=model_info.tokenizer_mode,
878
+ trust_remote_code=model_info.trust_remote_code,
879
+ hf_overrides=model_info.hf_overrides,
880
+ )
881
882
tokenizer_group = TokenizerGroup(
883
model,
884
enable_lora=False,
885
max_num_seqs=5,
886
max_input_length=None,
887
+ trust_remote_code=model_config.trust_remote_code,
888
)
889
tokenizer = tokenizer_group.tokenizer
890
891
# Test detecting the tokenizer's chat_template
892
chat_template = resolve_hf_chat_template(
893
+ model_config,
894
tokenizer,
895
chat_template=None,
896
tools=None,
877
- trust_remote_code=True,
897
)
898
assert isinstance(chat_template, str)
899
@@ -884,11 +903,70 @@ def test_resolve_content_format_hf_defined(model, expected_format):
903
print(_try_extract_ast(chat_template))
904
905
resolved_format = resolve_chat_template_content_format(
906
+ model_config,
907
+ None, # Test detecting the tokenizer's chat_template
908
+ None,
909
+ "auto",
910
+ tokenizer,
911
+ )
912
+
913
+ assert resolved_format == expected_format
914
+
915
+
916
+# yapf: disable
917
+@pytest.mark.parametrize(
918
+ ("model", "expected_format"),
919
+ [("Salesforce/blip2-opt-2.7b", "string"),
920
+ ("facebook/chameleon-7b", "string"),
921
+ ("deepseek-ai/deepseek-vl2-tiny", "string"),
922
+ ("microsoft/Florence-2-base", "string"),
923
+ ("adept/fuyu-8b", "string"),
924
+ ("google/paligemma-3b-mix-224", "string"),
925
+ ("Qwen/Qwen-VL", "string"),
926
+ ("Qwen/Qwen-VL-Chat", "string")],
927
+)
928
+# yapf: enable
929
+def test_resolve_content_format_fallbacks(model, expected_format):
930
+ model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
931
+ model_info.check_available_online(on_fail="skip")
932
+
933
+ model_config = ModelConfig(
934
+ model,
935
+ tokenizer=model_info.tokenizer or model,
936
+ tokenizer_mode=model_info.tokenizer_mode,
937
+ trust_remote_code=model_info.trust_remote_code,
938
+ hf_overrides=model_info.hf_overrides,
939
+ )
940
+
941
+ tokenizer_group = TokenizerGroup(
942
+ model_config.tokenizer,
943
+ enable_lora=False,
944
+ max_num_seqs=5,
945
+ max_input_length=None,
946
+ trust_remote_code=model_config.trust_remote_code,
947
+ )
948
+ tokenizer = tokenizer_group.tokenizer
949
+
950
+ # Test detecting the tokenizer's chat_template
951
+ chat_template = resolve_hf_chat_template(
952
+ model_config,
953
+ tokenizer,
954
+ chat_template=None,
955
+ tools=None,
956
+ )
957
+ assert isinstance(chat_template, str)
958
+
959
+ print("[TEXT]")
960
+ print(chat_template)
961
+ print("[AST]")
962
+ print(_try_extract_ast(chat_template))
963
+
964
+ resolved_format = resolve_chat_template_content_format(
965
+ model_config,
966
None, # Test detecting the tokenizer's chat_template
967
None,
968
"auto",
969
tokenizer,
891
- trust_remote_code=True,
970
)
971
972
assert resolved_format == expected_format
@@ -899,22 +977,14 @@ def test_resolve_content_format_hf_defined(model, expected_format):
977
("template_path", "expected_format"),
978
[("template_alpaca.jinja", "string"),
979
("template_baichuan.jinja", "string"),
902
- ("template_blip2.jinja", "string"),
903
- ("template_chameleon.jinja", "string"),
980
("template_chatglm.jinja", "string"),
981
("template_chatglm2.jinja", "string"),
982
("template_chatml.jinja", "string"),
907
- ("template_deepseek_vl2.jinja", "string"),
983
("template_dse_qwen2_vl.jinja", "openai"),
984
("template_falcon_180b.jinja", "string"),
985
("template_falcon.jinja", "string"),
911
- ("template_florence2.jinja", "string"),
912
- ("template_fuyu.jinja", "string"),
986
("template_inkbot.jinja", "string"),
914
- ("template_paligemma.jinja", "string"),
987
("template_teleflm.jinja", "string"),
916
- ("template_qwen_vl.jinja", "string"),
917
- ("template_qwen_vl_chat.jinja", "string"),
988
("template_vlm2vec.jinja", "openai"),
989
("tool_chat_template_granite_20b_fc.jinja", "string"),
990
("tool_chat_template_hermes.jinja", "string"),
@@ -926,11 +996,18 @@ def test_resolve_content_format_hf_defined(model, expected_format):
996
)
997
# yapf: enable
998
def test_resolve_content_format_examples(template_path, expected_format):
999
+ model_config = ModelConfig(
1000
+ PHI3V_MODEL_ID, # Dummy
1001
+ tokenizer=PHI3V_MODEL_ID, # Dummy
1002
+ trust_remote_code=True,
1003
+ )
1004
+
1005
tokenizer_group = TokenizerGroup(
930
- PHI3V_MODEL_ID,
1006
+ PHI3V_MODEL_ID, # Dummy
1007
enable_lora=False,
1008
max_num_seqs=5,
1009
max_input_length=None,
1010
+ trust_remote_code=model_config.trust_remote_code,
1011
)
1012
dummy_tokenizer = tokenizer_group.tokenizer
1013
dummy_tokenizer.chat_template = None
@@ -944,11 +1021,11 @@ def test_resolve_content_format_examples(template_path, expected_format):
1021
print(_try_extract_ast(chat_template))
1022
1023
resolved_format = resolve_chat_template_content_format(
1024
+ model_config,
1025
chat_template,
1026
None,
1027
"auto",
1028
dummy_tokenizer,
951
- trust_remote_code=True,
1029
)
1030
1031
assert resolved_format == expected_format
tests/models/registry.py
+4
-2
@@ -182,7 +182,9 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
182
"JAISLMHeadModel": _HfExamplesInfo("inceptionai/jais-13b-chat"),
183
"JambaForCausalLM": _HfExamplesInfo("ai21labs/AI21-Jamba-1.5-Mini",
184
extras={"tiny": "ai21labs/Jamba-tiny-dev"}), # noqa: E501
185
- "LlamaForCausalLM": _HfExamplesInfo("meta-llama/Llama-3.2-1B-Instruct"),
185
+ "LlamaForCausalLM": _HfExamplesInfo("meta-llama/Llama-3.2-1B-Instruct",
186
+ extras={"guard": "meta-llama/Llama-Guard-3-1B", # noqa: E501
187
+ "hermes": "NousResearch/Hermes-3-Llama-3.1-8B"}), # noqa: E501
188
"LLaMAForCausalLM": _HfExamplesInfo("decapoda-research/llama-7b-hf",
189
is_available_online=False),
190
"MambaForCausalLM": _HfExamplesInfo("state-spaces/mamba-130m-hf"),
@@ -378,7 +380,7 @@ _MULTIMODAL_EXAMPLE_MODELS = {
380
# Therefore, we borrow the BartTokenizer from the original Bart model
381
"Florence2ForConditionalGeneration": _HfExamplesInfo("microsoft/Florence-2-base", # noqa: E501
382
tokenizer="Isotr0py/Florence-2-tokenizer",
381
- trust_remote_code=True), # noqa: E501
383
+ trust_remote_code=True,), # noqa: E501
384
"MllamaForConditionalGeneration": _HfExamplesInfo("meta-llama/Llama-3.2-11B-Vision-Instruct"), # noqa: E501
385
"WhisperForConditionalGeneration": _HfExamplesInfo("openai/whisper-large-v3"), # noqa: E501
386
}
vllm/entrypoints/chat_utils.py
+26
-12
@@ -38,6 +38,10 @@ from vllm.config import ModelConfig
38
from vllm.logger import init_logger
39
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalDataDict
40
from vllm.multimodal.utils import MediaConnector
41
+# yapf: disable
42
+from vllm.transformers_utils.chat_templates import (
43
+ get_chat_template_fallback_path)
44
+# yapf: enable
45
from vllm.transformers_utils.processor import cached_get_processor
46
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
47
@@ -325,11 +329,10 @@ def resolve_mistral_chat_template(
329
return None
330
331
def resolve_hf_chat_template(
332
+ model_config: ModelConfig,
333
tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
334
chat_template: Optional[str],
335
tools: Optional[list[dict[str, Any]]],
331
- *,
332
- trust_remote_code: bool,
336
) -> Optional[str]:
337
# 1st priority: The given chat template
338
if chat_template is not None:
@@ -342,7 +345,7 @@ def resolve_hf_chat_template(
345
tokenizer.name_or_path,
346
processor_cls=(PreTrainedTokenizer, PreTrainedTokenizerFast,
347
ProcessorMixin),
345
- trust_remote_code=trust_remote_code,
348
+ trust_remote_code=model_config.trust_remote_code,
349
)
350
if isinstance(processor, ProcessorMixin) and \
351
processor.chat_template is not None:
@@ -358,22 +361,34 @@ def resolve_hf_chat_template(
361
logger.debug("Failed to load AutoTokenizer chat template for %s",
362
tokenizer.name_or_path, exc_info=True)
363
361
- return None
364
+ # 4th priority: Predefined fallbacks
365
+ path = get_chat_template_fallback_path(
366
+ model_type=model_config.hf_config.model_type,
367
+ tokenizer_name_or_path=model_config.tokenizer,
368
+ )
369
+ if path is not None:
370
+ logger.info("Loading chat template fallback for %s as there isn't one "
371
+ "defined on HF Hub.", tokenizer.name_or_path)
372
+ chat_template = load_chat_template(path)
373
+ else:
374
+ logger.debug("There is no chat template fallback for %s",
375
+ tokenizer.name_or_path)
376
+
377
+ return chat_template
378
379
380
def _resolve_chat_template_content_format(
381
+ model_config: ModelConfig,
382
chat_template: Optional[str],
383
tools: Optional[list[dict[str, Any]]],
384
given_format: ChatTemplateContentFormatOption,
385
tokenizer: AnyTokenizer,
369
- *,
370
- trust_remote_code: bool,
386
) -> _ChatTemplateContentFormat:
387
if isinstance(tokenizer, (PreTrainedTokenizer, PreTrainedTokenizerFast)):
388
hf_chat_template = resolve_hf_chat_template(
389
+ model_config,
390
tokenizer,
391
chat_template=chat_template,
376
- trust_remote_code=trust_remote_code,
392
tools=tools,
393
)
394
else:
@@ -413,19 +428,18 @@ def _log_chat_template_content_format(
428
429
430
def resolve_chat_template_content_format(
431
+ model_config: ModelConfig,
432
chat_template: Optional[str],
433
tools: Optional[list[dict[str, Any]]],
434
given_format: ChatTemplateContentFormatOption,
435
tokenizer: AnyTokenizer,
420
- *,
421
- trust_remote_code: bool = False,
436
) -> _ChatTemplateContentFormat:
437
detected_format = _resolve_chat_template_content_format(
438
+ model_config,
439
chat_template,
440
tools,
441
given_format,
442
tokenizer,
428
- trust_remote_code=trust_remote_code,
443
)
444
445
_log_chat_template_content_format(
@@ -1177,20 +1191,20 @@ def parse_chat_messages_futures(
1191
1192
1193
def apply_hf_chat_template(
1194
+ model_config: ModelConfig,
1195
tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
1196
conversation: list[ConversationMessage],
1197
chat_template: Optional[str],
1198
tools: Optional[list[dict[str, Any]]],
1199
*,
1185
- trust_remote_code: bool = False,
1200
tokenize: bool = False, # Different from HF's default
1201
**kwargs: Any,
1202
) -> str:
1203
hf_chat_template = resolve_hf_chat_template(
1204
+ model_config,
1205
tokenizer,
1206
chat_template=chat_template,
1207
tools=tools,
1193
- trust_remote_code=trust_remote_code,
1208
)
1209
1210
if hf_chat_template is None:
vllm/entrypoints/llm.py
+2
-2
@@ -726,11 +726,11 @@ class LLM:
726
tokenizer = self.get_tokenizer(lora_request)
727
model_config = self.llm_engine.get_model_config()
728
resolved_content_format = resolve_chat_template_content_format(
729
+ model_config,
730
chat_template,
731
tools,
732
chat_template_content_format,
733
tokenizer,
733
- trust_remote_code=model_config.trust_remote_code,
734
)
735
736
_chat_template_kwargs: dict[str, Any] = dict(
@@ -762,8 +762,8 @@ class LLM:
762
)
763
else:
764
prompt_str = apply_hf_chat_template(
765
+ model_config,
766
tokenizer,
766
- trust_remote_code=model_config.trust_remote_code,
767
conversation=conversation,
768
**_chat_template_kwargs,
769
)
vllm/entrypoints/openai/api_server.py
+2
-1
@@ -937,10 +937,11 @@ async def init_app_state(
937
chat_template=resolved_chat_template)
938
else:
939
hf_chat_template = resolve_hf_chat_template(
940
+ vllm_config.model_config,
941
tokenizer,
942
chat_template=None,
943
tools=None,
943
- trust_remote_code=model_config.trust_remote_code)
944
+ )
945
946
if hf_chat_template != resolved_chat_template:
947
logger.warning(
vllm/entrypoints/openai/serving_engine.py
+2
-2
@@ -394,11 +394,11 @@ class OpenAIServing:
394
model_config = self.model_config
395
396
resolved_content_format = resolve_chat_template_content_format(
397
+ model_config,
398
chat_template,
399
tool_dicts,
400
chat_template_content_format,
401
tokenizer,
401
- trust_remote_code=model_config.trust_remote_code,
402
)
403
conversation, mm_data_future = parse_chat_messages_futures(
404
messages,
@@ -425,8 +425,8 @@ class OpenAIServing:
425
)
426
else:
427
request_prompt = apply_hf_chat_template(
428
+ model_config,
429
tokenizer,
429
- trust_remote_code=model_config.trust_remote_code,
430
conversation=conversation,
431
**_chat_template_kwargs,
432
)
vllm/transformers_utils/chat_templates/__init__.py
new
+4
@@ -0,0 +1,4 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+from .registry import get_chat_template_fallback_path
3
+
4
+__all__ = ["get_chat_template_fallback_path"]
vllm/transformers_utils/chat_templates/registry.py
new
+59
@@ -0,0 +1,59 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+from pathlib import Path
3
+from typing import Callable, Optional, Union
4
+
5
+from vllm.logger import init_logger
6
+
7
+logger = init_logger(__file__)
8
+
9
+CHAT_TEMPLATES_DIR = Path(__file__).parent
10
+
11
+ChatTemplatePath = Union[Path, Callable[[str], Optional[Path]]]
12
+
13
+
14
+def _get_qwen_chat_template_fallback(
15
+ tokenizer_name_or_path: str) -> Optional[Path]:
16
+ if tokenizer_name_or_path.endswith("-Chat"):
17
+ return CHAT_TEMPLATES_DIR / "template_chatml.jinja"
18
+
19
+ return CHAT_TEMPLATES_DIR / "template_basic.jinja"
20
+
21
+
22
+# yapf: disable
23
+_MODEL_TYPE_TO_CHAT_TEMPLATE_FALLBACK: dict[str, ChatTemplatePath] = {
24
+ "blip-2": CHAT_TEMPLATES_DIR / "template_blip2.jinja",
25
+ "chameleon": CHAT_TEMPLATES_DIR / "template_basic.jinja",
26
+ "deepseek_vl_v2": CHAT_TEMPLATES_DIR / "template_deepseek_vl2.jinja",
27
+ "florence2": CHAT_TEMPLATES_DIR / "template_basic.jinja",
28
+ "fuyu": CHAT_TEMPLATES_DIR / "template_fuyu.jinja",
29
+ "paligemma": CHAT_TEMPLATES_DIR / "template_basic.jinja",
30
+ "qwen": _get_qwen_chat_template_fallback,
31
+}
32
+# yapf: enable
33
+
34
+
35
+def register_chat_template_fallback_path(
36
+ model_type: str,
37
+ chat_template: ChatTemplatePath,
38
+) -> None:
39
+ if model_type in _MODEL_TYPE_TO_CHAT_TEMPLATE_FALLBACK:
40
+ logger.warning(
41
+ "Model type %s already has a chat template registered. "
42
+ "It will be overwritten by the new chat template %s.", model_type,
43
+ chat_template)
44
+
45
+ _MODEL_TYPE_TO_CHAT_TEMPLATE_FALLBACK[model_type] = chat_template
46
+
47
+
48
+def get_chat_template_fallback_path(
49
+ model_type: str,
50
+ tokenizer_name_or_path: str,
51
+) -> Optional[Path]:
52
+ chat_template = _MODEL_TYPE_TO_CHAT_TEMPLATE_FALLBACK.get(model_type)
53
+ if callable(chat_template):
54
+ chat_template = chat_template(tokenizer_name_or_path)
55
+
56
+ if chat_template is None:
57
+ return None
58
+
59
+ return chat_template