| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import jinja2 |
| 5 | import pytest |
| 6 | |
| 7 | from vllm.config import ModelConfig |
| 8 | from vllm.entrypoints.chat_utils import load_chat_template |
| 9 | from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest |
| 10 | from vllm.exceptions import VLLMValidationError |
| 11 | from vllm.renderers.hf import ( |
| 12 | _consolidate_system_messages, |
| 13 | _convert_developer_to_system, |
| 14 | _detect_content_format, |
| 15 | _detect_developer_role_support, |
| 16 | _get_hf_base_chat_template_params, |
| 17 | _template_error_reason, |
| 18 | _try_extract_ast, |
| 19 | resolve_chat_template, |
| 20 | resolve_chat_template_content_format, |
| 21 | resolve_chat_template_kwargs, |
| 22 | safe_apply_chat_template, |
| 23 | ) |
| 24 | from vllm.tokenizers import get_tokenizer |
| 25 | |
| 26 | from ..models.registry import HF_EXAMPLE_MODELS |
| 27 | from ..utils import VLLM_PATH |
| 28 | |
| 29 | EXAMPLES_DIR = VLLM_PATH / "examples" |
| 30 | |
| 31 | chatml_jinja_path = VLLM_PATH / "examples/template_chatml.jinja" |
| 32 | assert chatml_jinja_path.exists() |
| 33 | |
| 34 | # Define models, templates, and their corresponding expected outputs |
| 35 | MODEL_TEMPLATE_GENERATION_OUTPUT = [ |
| 36 | ( |
| 37 | "facebook/opt-125m", |
| 38 | chatml_jinja_path, |
| 39 | True, |
| 40 | False, |
| 41 | """<|im_start|>user |
| 42 | Hello<|im_end|> |
| 43 | <|im_start|>assistant |
| 44 | Hi there!<|im_end|> |
| 45 | <|im_start|>user |
| 46 | What is the capital of<|im_end|> |
| 47 | <|im_start|>assistant |
| 48 | """, |
| 49 | ), |
| 50 | ( |
| 51 | "facebook/opt-125m", |
| 52 | chatml_jinja_path, |
| 53 | False, |
| 54 | False, |
| 55 | """<|im_start|>user |
| 56 | Hello<|im_end|> |
| 57 | <|im_start|>assistant |
| 58 | Hi there!<|im_end|> |
| 59 | <|im_start|>user |
| 60 | What is the capital of""", |
| 61 | ), |
| 62 | ( |
| 63 | "facebook/opt-125m", |
| 64 | chatml_jinja_path, |
| 65 | False, |
| 66 | True, |
| 67 | """<|im_start|>user |
| 68 | Hello<|im_end|> |
| 69 | <|im_start|>assistant |
| 70 | Hi there!<|im_end|> |
| 71 | <|im_start|>user |
| 72 | What is the capital of<|im_end|> |
| 73 | <|im_start|>assistant |
| 74 | The capital of""", |
| 75 | ), |
| 76 | ] |
| 77 | |
| 78 | TEST_MESSAGES = [ |
| 79 | {"role": "user", "content": "Hello"}, |
| 80 | {"role": "assistant", "content": "Hi there!"}, |
| 81 | {"role": "user", "content": "What is the capital of"}, |
| 82 | ] |
| 83 | ASSISTANT_MESSAGE_TO_CONTINUE = {"role": "assistant", "content": "The capital of"} |
| 84 | |
| 85 | |
| 86 | def test_load_chat_template(): |
| 87 | # Testing chatml template |
| 88 | template_content = load_chat_template(chat_template=chatml_jinja_path) |
| 89 | |
| 90 | # Test assertions |
| 91 | assert template_content is not None |
| 92 | # Hard coded value for template_chatml.jinja |
| 93 | assert ( |
| 94 | template_content |
| 95 | == """{% for message in messages %}{{'<|im_start|>' + message['role'] + '\\n' + message['content']}}{% if (loop.last and add_generation_prompt) or not loop.last %}{{ '<|im_end|>' + '\\n'}}{% endif %}{% endfor %} |
| 96 | {% if add_generation_prompt and messages[-1]['role'] != 'assistant' %}{{ '<|im_start|>assistant\\n' }}{% endif %}""" # noqa: E501 |
| 97 | ) |
| 98 | |
| 99 | |
| 100 | def test_no_load_chat_template_filelike(): |
| 101 | # Testing chatml template |
| 102 | template = "../../examples/does_not_exist" |
| 103 | |
| 104 | with pytest.raises(VLLMValidationError, match="looks like a file path"): |
| 105 | load_chat_template(chat_template=template) |
| 106 | |
| 107 | |
| 108 | def test_no_load_chat_template_literallike(): |
| 109 | # Testing chatml template |
| 110 | template = "{{ messages }}" |
| 111 | |
| 112 | template_content = load_chat_template(chat_template=template) |
| 113 | |
| 114 | assert template_content == template |
| 115 | |
| 116 | |
| 117 | @pytest.mark.parametrize( |
| 118 | "model", |
| 119 | [ |
| 120 | "Qwen/Qwen2-VL-2B-Instruct", # chat_template is of type str |
| 121 | "NousResearch/Hermes-3-Llama-3.1-8B", # chat_template is of type dict |
| 122 | ], |
| 123 | ) |
| 124 | @pytest.mark.parametrize("use_tools", [True, False]) |
| 125 | def test_resolve_chat_template(sample_json_schema, model, use_tools): |
| 126 | """Checks that chat_template is a dict type for HF models.""" |
| 127 | model_info = HF_EXAMPLE_MODELS.find_hf_info(model) |
| 128 | model_info.check_available_online(on_fail="skip") |
| 129 | |
| 130 | model_config = ModelConfig( |
| 131 | model, |
| 132 | tokenizer=model_info.tokenizer or model, |
| 133 | tokenizer_mode=model_info.tokenizer_mode, |
| 134 | revision=model_info.revision, |
| 135 | trust_remote_code=model_info.trust_remote_code, |
| 136 | hf_overrides=model_info.hf_overrides, |
| 137 | skip_tokenizer_init=model_info.require_embed_inputs, |
| 138 | enable_prompt_embeds=model_info.require_embed_inputs, |
| 139 | enable_mm_embeds=model_info.require_embed_inputs, |
| 140 | enforce_eager=model_info.enforce_eager, |
| 141 | dtype=model_info.dtype, |
| 142 | ) |
| 143 | |
| 144 | # Build the tokenizer |
| 145 | tokenizer = get_tokenizer( |
| 146 | model, |
| 147 | trust_remote_code=model_config.trust_remote_code, |
| 148 | ) |
| 149 | |
| 150 | tools = ( |
| 151 | [ |
| 152 | { |
| 153 | "type": "function", |
| 154 | "function": { |
| 155 | "name": "dummy_function_name", |
| 156 | "description": "This is a dummy function", |
| 157 | "parameters": sample_json_schema, |
| 158 | }, |
| 159 | } |
| 160 | ] |
| 161 | if use_tools |
| 162 | else None |
| 163 | ) |
| 164 | |
| 165 | # Test detecting the tokenizer's chat_template |
| 166 | chat_template = resolve_chat_template( |
| 167 | tokenizer, |
| 168 | chat_template=None, |
| 169 | tools=tools, |
| 170 | model_config=model_config, |
| 171 | ) |
| 172 | assert isinstance(chat_template, str) |
| 173 | |
| 174 | |
| 175 | @pytest.mark.parametrize( |
| 176 | "model, expected_kwargs", |
| 177 | [ |
| 178 | ( |
| 179 | "Qwen/Qwen2-VL-2B-Instruct", |
| 180 | { |
| 181 | "add_vision_id", |
| 182 | "add_generation_prompt", |
| 183 | "continue_final_message", |
| 184 | "tools", |
| 185 | }, |
| 186 | ), |
| 187 | ( |
| 188 | "Qwen/Qwen3-8B", |
| 189 | { |
| 190 | "enable_thinking", |
| 191 | "add_generation_prompt", |
| 192 | "continue_final_message", |
| 193 | "tools", |
| 194 | }, |
| 195 | ), |
| 196 | ], |
| 197 | ) |
| 198 | def test_resolve_chat_template_kwargs(sample_json_schema, model, expected_kwargs): |
| 199 | """Checks that chat_template is a dict type for HF models.""" |
| 200 | model_info = HF_EXAMPLE_MODELS.find_hf_info(model) |
| 201 | model_info.check_available_online(on_fail="skip") |
| 202 | |
| 203 | tools = [ |
| 204 | { |
| 205 | "type": "function", |
| 206 | "function": { |
| 207 | "name": "dummy_function_name", |
| 208 | "description": "This is a dummy function", |
| 209 | "parameters": sample_json_schema, |
| 210 | }, |
| 211 | } |
| 212 | ] |
| 213 | |
| 214 | chat_template_kwargs = { |
| 215 | # both unused |
| 216 | "unused_kwargs_1": 123, |
| 217 | "unused_kwargs_2": "abc", |
| 218 | # should not appear |
| 219 | "chat_template": "{% Hello world! %}", |
| 220 | "tokenize": True, |
| 221 | # used by tokenizer |
| 222 | "continue_final_message": True, |
| 223 | "tools": tools, |
| 224 | # both used by Qwen2-VL and Qwen3 |
| 225 | "add_generation_prompt": True, |
| 226 | # only used by Qwen2-VL |
| 227 | "add_vision_id": True, |
| 228 | # only used by Qwen3 |
| 229 | "enable_thinking": True, |
| 230 | } |
| 231 | |
| 232 | model_config = ModelConfig( |
| 233 | model, |
| 234 | tokenizer=model_info.tokenizer or model, |
| 235 | tokenizer_mode=model_info.tokenizer_mode, |
| 236 | revision=model_info.revision, |
| 237 | trust_remote_code=model_info.trust_remote_code, |
| 238 | hf_overrides=model_info.hf_overrides, |
| 239 | skip_tokenizer_init=model_info.require_embed_inputs, |
| 240 | enable_prompt_embeds=model_info.require_embed_inputs, |
| 241 | enable_mm_embeds=model_info.require_embed_inputs, |
| 242 | enforce_eager=model_info.enforce_eager, |
| 243 | dtype=model_info.dtype, |
| 244 | ) |
| 245 | |
| 246 | # Build the tokenizer |
| 247 | tokenizer = get_tokenizer( |
| 248 | model, |
| 249 | trust_remote_code=model_config.trust_remote_code, |
| 250 | ) |
| 251 | |
| 252 | # Test detecting the tokenizer's chat_template |
| 253 | chat_template = resolve_chat_template( |
| 254 | tokenizer, |
| 255 | chat_template=None, |
| 256 | tools=tools, |
| 257 | model_config=model_config, |
| 258 | ) |
| 259 | with pytest.raises( |
| 260 | ValueError, match="Found unexpected chat template kwargs from request" |
| 261 | ): |
| 262 | # should raise error if `chat_template_kwargs` contains |
| 263 | # `chat_template` or `tokenize` |
| 264 | resolve_chat_template_kwargs( |
| 265 | tokenizer, |
| 266 | chat_template=chat_template, |
| 267 | chat_template_kwargs=chat_template_kwargs, |
| 268 | ) |
| 269 | resolved_chat_template_kwargs = resolve_chat_template_kwargs( |
| 270 | tokenizer, |
| 271 | chat_template=chat_template, |
| 272 | chat_template_kwargs=chat_template_kwargs, |
| 273 | raise_on_unexpected=False, |
| 274 | ) |
| 275 | assert set(resolved_chat_template_kwargs.keys()) == expected_kwargs |
| 276 | |
| 277 | # Additional test: Verify HF base parameters work with **kwargs tokenizers |
| 278 | # This validates the fix for tokenizers like Kimi K2 that use **kwargs |
| 279 | # to receive standard HuggingFace parameters instead of declaring them explicitly |
| 280 | hf_base_params = _get_hf_base_chat_template_params() |
| 281 | # Verify common HF parameters are in the base class |
| 282 | assert {"add_generation_prompt", "tools", "continue_final_message"}.issubset( |
| 283 | hf_base_params |
| 284 | ), f"Expected HF base params not found in {hf_base_params}" |
| 285 | |
| 286 | # Test with a mock tokenizer that uses **kwargs (like Kimi K2) |
| 287 | class MockTokenizerWithKwargs: |
| 288 | def apply_chat_template(self, conversation, **kwargs): |
| 289 | return "mocked_output" |
| 290 | |
| 291 | mock_tokenizer = MockTokenizerWithKwargs() |
| 292 | mock_kwargs = { |
| 293 | "add_generation_prompt": True, |
| 294 | "tools": tools, |
| 295 | "continue_final_message": False, |
| 296 | "unknown_param": "should_be_filtered", |
| 297 | } |
| 298 | resolved_mock = resolve_chat_template_kwargs( |
| 299 | mock_tokenizer, chat_template, mock_kwargs, raise_on_unexpected=False |
| 300 | ) |
| 301 | # HF base params should pass through even with **kwargs tokenizer |
| 302 | assert "add_generation_prompt" in resolved_mock |
| 303 | assert "tools" in resolved_mock |
| 304 | assert "continue_final_message" in resolved_mock |
| 305 | # Unknown params should be filtered out |
| 306 | assert "unknown_param" not in resolved_mock |
| 307 | |
| 308 | |
| 309 | def test_resolve_chat_template_resolves_name(): |
| 310 | """When chat_template is a name, resolve_chat_template should return |
| 311 | the actual Jinja content so that kwargs detection works correctly.""" |
| 312 | from unittest.mock import MagicMock |
| 313 | |
| 314 | jinja_content = "{{ messages }}{% if tools %}{{ tools }}{% endif %}" |
| 315 | tokenizer = MagicMock() |
| 316 | tokenizer.get_chat_template.return_value = jinja_content |
| 317 | |
| 318 | model_config = MagicMock() |
| 319 | |
| 320 | result = resolve_chat_template( |
| 321 | tokenizer, |
| 322 | chat_template="tool_use", |
| 323 | tools=None, |
| 324 | model_config=model_config, |
| 325 | ) |
| 326 | |
| 327 | assert result == jinja_content |
| 328 | tokenizer.get_chat_template.assert_called_once_with("tool_use", tools=None) |
| 329 | |
| 330 | |
| 331 | def test_resolve_chat_template_kwargs_with_template_name(): |
| 332 | """Ensures template kwargs are not silently dropped when chat_template |
| 333 | was originally a template name that has been resolved to Jinja content.""" |
| 334 | from unittest.mock import MagicMock |
| 335 | |
| 336 | jinja_content = ( |
| 337 | "{% for m in messages %}{{ m }}{% endfor %}" |
| 338 | "{% if tools %}{{ tools }}{% endif %}" |
| 339 | "{% if documents %}{{ documents }}{% endif %}" |
| 340 | ) |
| 341 | |
| 342 | tokenizer = MagicMock() |
| 343 | tokenizer.apply_chat_template = MagicMock() |
| 344 | |
| 345 | kwargs = { |
| 346 | "tools": [{"type": "function", "function": {"name": "f"}}], |
| 347 | "documents": [{"title": "doc"}], |
| 348 | "unknown_param": "should be dropped", |
| 349 | } |
| 350 | |
| 351 | resolved = resolve_chat_template_kwargs( |
| 352 | tokenizer, |
| 353 | chat_template=jinja_content, |
| 354 | chat_template_kwargs=kwargs, |
| 355 | raise_on_unexpected=False, |
| 356 | ) |
| 357 | |
| 358 | # template vars "tools" and "documents" should be preserved |
| 359 | assert "tools" in resolved |
| 360 | assert "documents" in resolved |
| 361 | # unknown param should be filtered |
| 362 | assert "unknown_param" not in resolved |
| 363 | |
| 364 | |
| 365 | @pytest.mark.parametrize( |
| 366 | ("chat_template", "expected_format"), |
| 367 | [ |
| 368 | ( |
| 369 | """ |
| 370 | {% macro render_message(role, message_content) %} |
| 371 | {% for item in message_content %}{{ item['type'] }}{% endfor %} |
| 372 | {% endmacro %} |
| 373 | {% for message in messages %} |
| 374 | {{ render_message(message['role'], message['content']) }} |
| 375 | {% endfor %} |
| 376 | """, |
| 377 | "openai", |
| 378 | ), |
| 379 | ( |
| 380 | """ |
| 381 | {% macro render_message(role, message_content) %} |
| 382 | {% for item in message_content %}{{ item['type'] }}{% endfor %} |
| 383 | {% endmacro %} |
| 384 | {% for message in messages %} |
| 385 | {{ render_message( |
| 386 | role=message['role'], |
| 387 | message_content=message['content'], |
| 388 | ) }} |
| 389 | {% endfor %} |
| 390 | """, |
| 391 | "openai", |
| 392 | ), |
| 393 | ( |
| 394 | """ |
| 395 | {% for message in messages %} |
| 396 | {% for item in message['content'] %}{{ item['type'] }}{% endfor %} |
| 397 | {% endfor %} |
| 398 | """, |
| 399 | "openai", |
| 400 | ), |
| 401 | ( |
| 402 | """ |
| 403 | {% macro render_message(content) %} |
| 404 | {% for item in content %}{{ item['type'] }}{% endfor %} |
| 405 | {% endmacro %} |
| 406 | {% for message in messages %} |
| 407 | {{ render_message(content=['text only']) }} |
| 408 | {% endfor %} |
| 409 | """, |
| 410 | "string", |
| 411 | ), |
| 412 | ( |
| 413 | """ |
| 414 | {% for message in messages %}{{ message['content'] }}{% endfor %} |
| 415 | """, |
| 416 | "string", |
| 417 | ), |
| 418 | ], |
| 419 | ) |
| 420 | def test_detect_content_format(chat_template, expected_format): |
| 421 | """Detect content format when content is passed through a macro.""" |
| 422 | assert _detect_content_format(chat_template, default="string") == expected_format |
| 423 | |
| 424 | |
| 425 | # NOTE: Qwen2-Audio default chat template is specially defined inside |
| 426 | # processor class instead of using `tokenizer_config.json` |
| 427 | @pytest.mark.parametrize( |
| 428 | ("model", "expected_format"), |
| 429 | [ |
| 430 | ("microsoft/Phi-3.5-vision-instruct", "string"), |
| 431 | ("Qwen/Qwen2-VL-2B-Instruct", "openai"), |
| 432 | ("Qwen/Qwen2.5-VL-3B-Instruct", "openai"), |
| 433 | ("Qwen/Qwen3.5-4B", "openai"), |
| 434 | ("fixie-ai/ultravox-v0_5-llama-3_2-1b", "string"), |
| 435 | ("Qwen/Qwen2-Audio-7B-Instruct", "openai"), |
| 436 | ("meta-llama/Llama-Guard-3-1B", "openai"), |
| 437 | ("XiaomiMiMo/MiMo-V2.5-Pro", "openai"), |
| 438 | ], |
| 439 | ) |
| 440 | def test_resolve_content_format_hf_defined(model, expected_format): |
| 441 | """Detect the chat template content format for built-in HF models.""" |
| 442 | model_info = HF_EXAMPLE_MODELS.find_hf_info(model) |
| 443 | model_info.check_available_online(on_fail="skip") |
| 444 | |
| 445 | model_config = ModelConfig( |
| 446 | model, |
| 447 | tokenizer=model_info.tokenizer or model, |
| 448 | tokenizer_mode=model_info.tokenizer_mode, |
| 449 | revision=model_info.revision, |
| 450 | trust_remote_code=model_info.trust_remote_code, |
| 451 | hf_overrides=model_info.hf_overrides, |
| 452 | skip_tokenizer_init=model_info.require_embed_inputs, |
| 453 | enable_prompt_embeds=model_info.require_embed_inputs, |
| 454 | enable_mm_embeds=model_info.require_embed_inputs, |
| 455 | enforce_eager=model_info.enforce_eager, |
| 456 | dtype=model_info.dtype, |
| 457 | ) |
| 458 | |
| 459 | tokenizer = get_tokenizer( |
| 460 | model, |
| 461 | trust_remote_code=model_config.trust_remote_code, |
| 462 | ) |
| 463 | |
| 464 | # Test detecting the tokenizer's chat_template |
| 465 | chat_template = resolve_chat_template( |
| 466 | tokenizer, |
| 467 | chat_template=None, |
| 468 | tools=None, |
| 469 | model_config=model_config, |
| 470 | ) |
| 471 | assert isinstance(chat_template, str) |
| 472 | |
| 473 | print("[TEXT]") |
| 474 | print(chat_template) |
| 475 | print("[AST]") |
| 476 | print(_try_extract_ast(chat_template)) |
| 477 | |
| 478 | resolved_format = resolve_chat_template_content_format( |
| 479 | None, # Test detecting the tokenizer's chat_template |
| 480 | None, |
| 481 | "auto", |
| 482 | tokenizer, |
| 483 | model_config=model_config, |
| 484 | ) |
| 485 | |
| 486 | assert resolved_format == expected_format |
| 487 | |
| 488 | |
| 489 | @pytest.mark.parametrize( |
| 490 | ("model", "expected_format"), |
| 491 | [ |
| 492 | ("Salesforce/blip2-opt-2.7b", "string"), |
| 493 | ("deepseek-ai/deepseek-vl2-tiny", "string"), |
| 494 | ("google/paligemma-3b-mix-224", "string"), |
| 495 | ], |
| 496 | ) |
| 497 | def test_resolve_content_format_fallbacks(model, expected_format): |
| 498 | model_info = HF_EXAMPLE_MODELS.find_hf_info(model) |
| 499 | model_info.check_available_online(on_fail="skip") |
| 500 | |
| 501 | model_config = ModelConfig( |
| 502 | model, |
| 503 | tokenizer=model_info.tokenizer or model, |
| 504 | tokenizer_mode=model_info.tokenizer_mode, |
| 505 | revision=model_info.revision, |
| 506 | trust_remote_code=model_info.trust_remote_code, |
| 507 | hf_overrides=model_info.hf_overrides, |
| 508 | skip_tokenizer_init=model_info.require_embed_inputs, |
| 509 | enable_prompt_embeds=model_info.require_embed_inputs, |
| 510 | enable_mm_embeds=model_info.require_embed_inputs, |
| 511 | enforce_eager=model_info.enforce_eager, |
| 512 | dtype=model_info.dtype, |
| 513 | ) |
| 514 | |
| 515 | tokenizer = get_tokenizer( |
| 516 | model_config.tokenizer, |
| 517 | trust_remote_code=model_config.trust_remote_code, |
| 518 | ) |
| 519 | |
| 520 | # Test detecting the tokenizer's chat_template |
| 521 | chat_template = resolve_chat_template( |
| 522 | tokenizer, |
| 523 | chat_template=None, |
| 524 | tools=None, |
| 525 | model_config=model_config, |
| 526 | ) |
| 527 | assert isinstance(chat_template, str) |
| 528 | |
| 529 | print("[TEXT]") |
| 530 | print(chat_template) |
| 531 | print("[AST]") |
| 532 | print(_try_extract_ast(chat_template)) |
| 533 | |
| 534 | resolved_format = resolve_chat_template_content_format( |
| 535 | None, # Test detecting the tokenizer's chat_template |
| 536 | None, |
| 537 | "auto", |
| 538 | tokenizer, |
| 539 | model_config=model_config, |
| 540 | ) |
| 541 | |
| 542 | assert resolved_format == expected_format |
| 543 | |
| 544 | |
| 545 | @pytest.mark.parametrize( |
| 546 | ("template_path", "expected_format"), |
| 547 | [ |
| 548 | ("template_alpaca.jinja", "string"), |
| 549 | ("template_chatglm.jinja", "string"), |
| 550 | ("template_chatglm2.jinja", "string"), |
| 551 | ("template_chatml.jinja", "string"), |
| 552 | ("template_falcon_180b.jinja", "string"), |
| 553 | ("template_falcon.jinja", "string"), |
| 554 | ("template_inkbot.jinja", "string"), |
| 555 | ("template_teleflm.jinja", "string"), |
| 556 | ("pooling/embed/template/dse_qwen2_vl.jinja", "openai"), |
| 557 | ("pooling/embed/template/vlm2vec_phi3v.jinja", "openai"), |
| 558 | ("pooling/embed/template/vlm2vec_qwen2vl.jinja", "openai"), |
| 559 | ("tool_chat_template_granite_20b_fc.jinja", "string"), |
| 560 | ("tool_chat_template_hermes.jinja", "string"), |
| 561 | ("tool_chat_template_internlm2_tool.jinja", "string"), |
| 562 | ("tool_chat_template_llama3.1_json.jinja", "openai"), |
| 563 | ("tool_chat_template_llama3.2_json.jinja", "openai"), |
| 564 | ("tool_chat_template_mistral_parallel.jinja", "string"), |
| 565 | ("tool_chat_template_mistral.jinja", "string"), |
| 566 | ], |
| 567 | ) |
| 568 | def test_resolve_content_format_examples(template_path, expected_format): |
| 569 | model = "Qwen/Qwen2-VL-2B-Instruct" # Dummy |
| 570 | model_config = ModelConfig( |
| 571 | model, |
| 572 | tokenizer=model, |
| 573 | trust_remote_code=True, |
| 574 | ) |
| 575 | |
| 576 | dummy_tokenizer = get_tokenizer( |
| 577 | model, |
| 578 | trust_remote_code=model_config.trust_remote_code, |
| 579 | ) |
| 580 | dummy_tokenizer.chat_template = None |
| 581 | |
| 582 | chat_template = load_chat_template(EXAMPLES_DIR / template_path) |
| 583 | assert isinstance(chat_template, str) |
| 584 | |
| 585 | print("[TEXT]") |
| 586 | print(chat_template) |
| 587 | print("[AST]") |
| 588 | print(_try_extract_ast(chat_template)) |
| 589 | |
| 590 | resolved_format = resolve_chat_template_content_format( |
| 591 | chat_template, |
| 592 | None, |
| 593 | "auto", |
| 594 | dummy_tokenizer, |
| 595 | model_config=model_config, |
| 596 | ) |
| 597 | |
| 598 | assert resolved_format == expected_format |
| 599 | |
| 600 | |
| 601 | @pytest.mark.parametrize( |
| 602 | "model,template,add_generation_prompt,continue_final_message,expected_output", |
| 603 | MODEL_TEMPLATE_GENERATION_OUTPUT, |
| 604 | ) |
| 605 | def test_get_gen_prompt( |
| 606 | model, template, add_generation_prompt, continue_final_message, expected_output |
| 607 | ): |
| 608 | model_info = HF_EXAMPLE_MODELS.find_hf_info(model) |
| 609 | model_info.check_available_online(on_fail="skip") |
| 610 | |
| 611 | model_config = ModelConfig( |
| 612 | model, |
| 613 | tokenizer=model_info.tokenizer or model, |
| 614 | tokenizer_mode=model_info.tokenizer_mode, |
| 615 | trust_remote_code=model_info.trust_remote_code, |
| 616 | revision=model_info.revision, |
| 617 | hf_overrides=model_info.hf_overrides, |
| 618 | skip_tokenizer_init=model_info.require_embed_inputs, |
| 619 | enable_prompt_embeds=model_info.require_embed_inputs, |
| 620 | enable_mm_embeds=model_info.require_embed_inputs, |
| 621 | enforce_eager=model_info.enforce_eager, |
| 622 | dtype=model_info.dtype, |
| 623 | ) |
| 624 | |
| 625 | # Initialize the tokenizer |
| 626 | tokenizer = get_tokenizer( |
| 627 | tokenizer_name=model_config.tokenizer, |
| 628 | trust_remote_code=model_config.trust_remote_code, |
| 629 | ) |
| 630 | template_content = load_chat_template(chat_template=template) |
| 631 | |
| 632 | # Create a mock request object using keyword arguments |
| 633 | mock_request = ChatCompletionRequest( |
| 634 | model=model, |
| 635 | messages=TEST_MESSAGES + [ASSISTANT_MESSAGE_TO_CONTINUE] |
| 636 | if continue_final_message |
| 637 | else TEST_MESSAGES, |
| 638 | add_generation_prompt=add_generation_prompt, |
| 639 | continue_final_message=continue_final_message, |
| 640 | ) |
| 641 | |
| 642 | # Call the function and get the result |
| 643 | result = safe_apply_chat_template( |
| 644 | model_config, |
| 645 | tokenizer, |
| 646 | mock_request.messages, |
| 647 | tools=None, |
| 648 | chat_template=mock_request.chat_template or template_content, |
| 649 | add_generation_prompt=mock_request.add_generation_prompt, |
| 650 | continue_final_message=mock_request.continue_final_message, |
| 651 | tokenize=False, |
| 652 | ) |
| 653 | |
| 654 | # Test assertion |
| 655 | assert result == expected_output, ( |
| 656 | f"The generated prompt does not match the expected output for " |
| 657 | f"model {model} and template {template}" |
| 658 | ) |
| 659 | |
| 660 | |
| 661 | class TestConvertDeveloperToSystem: |
| 662 | def test_converts_role(self): |
| 663 | conversation = [ |
| 664 | {"role": "developer", "content": "You are helpful."}, |
| 665 | {"role": "user", "content": "Hello"}, |
| 666 | ] |
| 667 | result = _convert_developer_to_system(conversation) |
| 668 | assert result[0]["role"] == "system" |
| 669 | assert result[0]["content"] == "You are helpful." |
| 670 | assert result[1]["role"] == "user" |
| 671 | |
| 672 | def test_removes_tools_key(self): |
| 673 | conversation = [ |
| 674 | { |
| 675 | "role": "developer", |
| 676 | "content": "Instructions", |
| 677 | "tools": [{"type": "function"}], |
| 678 | }, |
| 679 | ] |
| 680 | result = _convert_developer_to_system(conversation) |
| 681 | assert "tools" not in result[0] |
| 682 | |
| 683 | def test_no_developer_messages_unchanged(self): |
| 684 | conversation = [ |
| 685 | {"role": "system", "content": "System prompt"}, |
| 686 | {"role": "user", "content": "Hello"}, |
| 687 | ] |
| 688 | result = _convert_developer_to_system(conversation) |
| 689 | assert result[0]["role"] == "system" |
| 690 | assert result[1]["role"] == "user" |
| 691 | |
| 692 | def test_does_not_mutate_original(self): |
| 693 | original = { |
| 694 | "role": "developer", |
| 695 | "content": "Instructions", |
| 696 | "tools": [{"type": "function"}], |
| 697 | } |
| 698 | conversation = [original] |
| 699 | _convert_developer_to_system(conversation) |
| 700 | assert original["role"] == "developer" |
| 701 | assert "tools" in original |
| 702 | |
| 703 | |
| 704 | # --- Developer role detection and conversion tests --- |
| 705 | |
| 706 | CHATML_TEMPLATE = ( |
| 707 | "{% for message in messages %}" |
| 708 | "{{'<|im_start|>' + message['role'] + '\\n' + message['content']}}" |
| 709 | "{% if (loop.last and add_generation_prompt) or not loop.last %}" |
| 710 | "{{ '<|im_end|>' + '\\n'}}" |
| 711 | "{% endif %}" |
| 712 | "{% endfor %}" |
| 713 | "{% if add_generation_prompt and messages[-1]['role'] != 'assistant' %}" |
| 714 | "{{ '<|im_start|>assistant\\n' }}" |
| 715 | "{% endif %}" |
| 716 | ) |
| 717 | |
| 718 | TEMPLATE_WITH_DEVELOPER = ( |
| 719 | "{% for message in messages %}" |
| 720 | "{% if message['role'] == 'developer' %}" |
| 721 | "{{'<|im_start|>developer\\n' + message['content'] + '<|im_end|>\\n'}}" |
| 722 | "{% elif message['role'] == 'system' %}" |
| 723 | "{{'<|im_start|>system\\n' + message['content'] + '<|im_end|>\\n'}}" |
| 724 | "{% elif message['role'] == 'user' %}" |
| 725 | "{{'<|im_start|>user\\n' + message['content'] + '<|im_end|>\\n'}}" |
| 726 | "{% elif message['role'] == 'assistant' %}" |
| 727 | "{{'<|im_start|>assistant\\n' + message['content'] + '<|im_end|>\\n'}}" |
| 728 | "{% endif %}" |
| 729 | "{% endfor %}" |
| 730 | "{% if add_generation_prompt %}" |
| 731 | "{{ '<|im_start|>assistant\\n' }}" |
| 732 | "{% endif %}" |
| 733 | ) |
| 734 | |
| 735 | STRICT_ROLE_TEMPLATE = ( |
| 736 | "{% for message in messages %}" |
| 737 | "{% if message['role'] == 'system' %}" |
| 738 | "{{'<|im_start|>system\\n' + message['content'] + '<|im_end|>\\n'}}" |
| 739 | "{% elif message['role'] == 'user' %}" |
| 740 | "{{'<|im_start|>user\\n' + message['content'] + '<|im_end|>\\n'}}" |
| 741 | "{% elif message['role'] == 'assistant' %}" |
| 742 | "{{'<|im_start|>assistant\\n' + message['content'] + '<|im_end|>\\n'}}" |
| 743 | "{% else %}" |
| 744 | "{{ raise_exception('Unexpected message role: ' + message['role']) }}" |
| 745 | "{% endif %}" |
| 746 | "{% endfor %}" |
| 747 | "{% if add_generation_prompt %}" |
| 748 | "{{ '<|im_start|>assistant\\n' }}" |
| 749 | "{% endif %}" |
| 750 | ) |
| 751 | |
| 752 | |
| 753 | class TestDetectDeveloperRoleSupport: |
| 754 | def test_absent_in_chatml(self): |
| 755 | assert _detect_developer_role_support(CHATML_TEMPLATE) is False |
| 756 | |
| 757 | def test_present_double_quotes(self): |
| 758 | assert _detect_developer_role_support(TEMPLATE_WITH_DEVELOPER) is True |
| 759 | |
| 760 | def test_present_single_quotes(self): |
| 761 | template = TEMPLATE_WITH_DEVELOPER.replace('"developer"', "'developer'") |
| 762 | assert _detect_developer_role_support(template) is True |
| 763 | |
| 764 | def test_absent_in_strict_template(self): |
| 765 | assert _detect_developer_role_support(STRICT_ROLE_TEMPLATE) is False |
| 766 | |
| 767 | |
| 768 | class TestSafeApplyChatTemplateDeveloperRole: |
| 769 | @pytest.fixture |
| 770 | def model_config(self): |
| 771 | return ModelConfig( |
| 772 | "facebook/opt-125m", |
| 773 | tokenizer="facebook/opt-125m", |
| 774 | tokenizer_mode="auto", |
| 775 | trust_remote_code=False, |
| 776 | dtype="float16", |
| 777 | ) |
| 778 | |
| 779 | @pytest.fixture |
| 780 | def tokenizer(self): |
| 781 | return get_tokenizer("facebook/opt-125m") |
| 782 | |
| 783 | def test_developer_converted_to_system_for_chatml(self, model_config, tokenizer): |
| 784 | conversation = [ |
| 785 | {"role": "developer", "content": "You are a helpful assistant."}, |
| 786 | {"role": "user", "content": "Hello"}, |
| 787 | ] |
| 788 | result = safe_apply_chat_template( |
| 789 | model_config, |
| 790 | tokenizer, |
| 791 | conversation, |
| 792 | chat_template=CHATML_TEMPLATE, |
| 793 | tokenize=False, |
| 794 | add_generation_prompt=True, |
| 795 | ) |
| 796 | assert "<|im_start|>system" in result |
| 797 | assert "You are a helpful assistant." in result |
| 798 | assert "<|im_start|>developer" not in result |
| 799 | |
| 800 | def test_developer_preserved_when_template_supports_it( |
| 801 | self, model_config, tokenizer |
| 802 | ): |
| 803 | conversation = [ |
| 804 | {"role": "developer", "content": "You are a helpful assistant."}, |
| 805 | {"role": "user", "content": "Hello"}, |
| 806 | ] |
| 807 | result = safe_apply_chat_template( |
| 808 | model_config, |
| 809 | tokenizer, |
| 810 | conversation, |
| 811 | chat_template=TEMPLATE_WITH_DEVELOPER, |
| 812 | tokenize=False, |
| 813 | add_generation_prompt=True, |
| 814 | ) |
| 815 | assert "<|im_start|>developer" in result |
| 816 | assert "You are a helpful assistant." in result |
| 817 | |
| 818 | def test_developer_does_not_crash_strict_template(self, model_config, tokenizer): |
| 819 | conversation = [ |
| 820 | {"role": "developer", "content": "You are a helpful assistant."}, |
| 821 | {"role": "user", "content": "Hello"}, |
| 822 | ] |
| 823 | result = safe_apply_chat_template( |
| 824 | model_config, |
| 825 | tokenizer, |
| 826 | conversation, |
| 827 | chat_template=STRICT_ROLE_TEMPLATE, |
| 828 | tokenize=False, |
| 829 | add_generation_prompt=True, |
| 830 | ) |
| 831 | assert "<|im_start|>system" in result |
| 832 | assert "You are a helpful assistant." in result |
| 833 | |
| 834 | def test_no_developer_messages_no_overhead(self, model_config, tokenizer): |
| 835 | conversation = [ |
| 836 | {"role": "system", "content": "You are helpful."}, |
| 837 | {"role": "user", "content": "Hello"}, |
| 838 | ] |
| 839 | result = safe_apply_chat_template( |
| 840 | model_config, |
| 841 | tokenizer, |
| 842 | conversation, |
| 843 | chat_template=CHATML_TEMPLATE, |
| 844 | tokenize=False, |
| 845 | add_generation_prompt=True, |
| 846 | ) |
| 847 | assert "<|im_start|>system" in result |
| 848 | assert "You are helpful." in result |
| 849 | |
| 850 | def test_developer_at_non_first_position_consolidated( |
| 851 | self, model_config, tokenizer |
| 852 | ): |
| 853 | conversation = [ |
| 854 | {"role": "system", "content": "You are helpful."}, |
| 855 | {"role": "user", "content": "Hello"}, |
| 856 | {"role": "assistant", "content": "Hi there!"}, |
| 857 | {"role": "developer", "content": "Be concise."}, |
| 858 | {"role": "user", "content": "What is 2+2?"}, |
| 859 | ] |
| 860 | result = safe_apply_chat_template( |
| 861 | model_config, |
| 862 | tokenizer, |
| 863 | conversation, |
| 864 | chat_template=SYSTEM_FIRST_TEMPLATE, |
| 865 | tokenize=False, |
| 866 | add_generation_prompt=True, |
| 867 | ) |
| 868 | assert "<|im_start|>system" in result |
| 869 | assert "You are helpful." in result |
| 870 | assert "Be concise." in result |
| 871 | assert "What is 2+2?" in result |
| 872 | |
| 873 | def test_developer_only_no_prior_system(self, model_config, tokenizer): |
| 874 | conversation = [ |
| 875 | {"role": "user", "content": "Hello"}, |
| 876 | {"role": "developer", "content": "Be concise."}, |
| 877 | {"role": "user", "content": "What is 2+2?"}, |
| 878 | ] |
| 879 | result = safe_apply_chat_template( |
| 880 | model_config, |
| 881 | tokenizer, |
| 882 | conversation, |
| 883 | chat_template=SYSTEM_FIRST_TEMPLATE, |
| 884 | tokenize=False, |
| 885 | add_generation_prompt=True, |
| 886 | ) |
| 887 | assert "<|im_start|>system" in result |
| 888 | assert "Be concise." in result |
| 889 | |
| 890 | |
| 891 | SYSTEM_FIRST_TEMPLATE = ( |
| 892 | "{% for message in messages %}" |
| 893 | "{% if message['role'] == 'system' %}" |
| 894 | "{% if not loop.first %}" |
| 895 | "{{ raise_exception('System message must be at the beginning.') }}" |
| 896 | "{% endif %}" |
| 897 | "{{'<|im_start|>system\\n' + message['content'] + '<|im_end|>\\n'}}" |
| 898 | "{% elif message['role'] == 'user' %}" |
| 899 | "{{'<|im_start|>user\\n' + message['content'] + '<|im_end|>\\n'}}" |
| 900 | "{% elif message['role'] == 'assistant' %}" |
| 901 | "{{'<|im_start|>assistant\\n' + message['content'] + '<|im_end|>\\n'}}" |
| 902 | "{% else %}" |
| 903 | "{{ raise_exception('Unexpected message role: ' + message['role']) }}" |
| 904 | "{% endif %}" |
| 905 | "{% endfor %}" |
| 906 | "{% if add_generation_prompt %}" |
| 907 | "{{ '<|im_start|>assistant\\n' }}" |
| 908 | "{% endif %}" |
| 909 | ) |
| 910 | |
| 911 | |
| 912 | EFFORT_VALIDATING_TEMPLATE = ( |
| 913 | "{% if reasoning_effort is defined and " |
| 914 | "reasoning_effort not in ['xhigh', 'medium', 'low'] %}" |
| 915 | "{{ raise_exception('Unexpected reasoning effort ' + reasoning_effort + " |
| 916 | "'. Supported types are xhigh (default), medium, and low.') }}" |
| 917 | "{% endif %}" |
| 918 | "{% for message in messages %}" |
| 919 | "{{ message['role'] }}: {{ message['content'] }}\n" |
| 920 | "{% endfor %}" |
| 921 | ) |
| 922 | |
| 923 | EFFORT_BREAKING_TEMPLATE = ( |
| 924 | "{% if reasoning_effort is defined %}" |
| 925 | "{{ raise_exception('Template broke for an unrelated reason') }}" |
| 926 | "{% endif %}" |
| 927 | "{% for message in messages %}" |
| 928 | "{{ message['role'] }}: {{ message['content'] }}\n" |
| 929 | "{% endfor %}" |
| 930 | ) |
| 931 | |
| 932 | |
| 933 | class TestApplyChatTemplateEffortTolerant: |
| 934 | """Chat templates that reject unsupported reasoning_effort values should |
| 935 | surface a 400-style client error instead of crashing the request.""" |
| 936 | |
| 937 | @pytest.fixture |
| 938 | def model_config(self): |
| 939 | return ModelConfig( |
| 940 | "facebook/opt-125m", |
| 941 | tokenizer="facebook/opt-125m", |
| 942 | tokenizer_mode="auto", |
| 943 | trust_remote_code=False, |
| 944 | dtype="float16", |
| 945 | ) |
| 946 | |
| 947 | @pytest.fixture |
| 948 | def tokenizer(self): |
| 949 | return get_tokenizer("facebook/opt-125m") |
| 950 | |
| 951 | def test_unsupported_effort_raises_bad_request(self, model_config, tokenizer): |
| 952 | conversation = [{"role": "user", "content": "Hello"}] |
| 953 | with pytest.raises( |
| 954 | VLLMValidationError, |
| 955 | match="Unexpected reasoning effort high", |
| 956 | ) as excinfo: |
| 957 | safe_apply_chat_template( |
| 958 | model_config, |
| 959 | tokenizer, |
| 960 | conversation, |
| 961 | chat_template=EFFORT_VALIDATING_TEMPLATE, |
| 962 | tokenize=False, |
| 963 | reasoning_effort="high", |
| 964 | ) |
| 965 | # The client sees exactly the template's own reason, with no wrapper |
| 966 | # noise. It tells them which values are supported. |
| 967 | assert str(excinfo.value) == ( |
| 968 | "Unexpected reasoning effort high. Supported types are xhigh " |
| 969 | "(default), medium, and low." |
| 970 | ) |
| 971 | |
| 972 | def test_supported_effort_accepted(self, model_config, tokenizer): |
| 973 | conversation = [{"role": "user", "content": "Hello"}] |
| 974 | result = safe_apply_chat_template( |
| 975 | model_config, |
| 976 | tokenizer, |
| 977 | conversation, |
| 978 | chat_template=EFFORT_VALIDATING_TEMPLATE, |
| 979 | tokenize=False, |
| 980 | reasoning_effort="medium", |
| 981 | ) |
| 982 | assert result == "user: Hello\n" |
| 983 | |
| 984 | def test_non_effort_template_error_is_bad_request(self, model_config, tokenizer): |
| 985 | conversation = [{"role": "user", "content": "Hello"}] |
| 986 | with pytest.raises(VLLMValidationError, match="unrelated reason"): |
| 987 | safe_apply_chat_template( |
| 988 | model_config, |
| 989 | tokenizer, |
| 990 | conversation, |
| 991 | chat_template=EFFORT_BREAKING_TEMPLATE, |
| 992 | tokenize=False, |
| 993 | reasoning_effort="high", |
| 994 | ) |
| 995 | |
| 996 | |
| 997 | def test_template_error_reason_prefers_template_error(): |
| 998 | # Upstream wrappers must not hide the template's own reason from clients. |
| 999 | reason = jinja2.TemplateError("Unexpected reasoning effort high") |
| 1000 | wrapper = ValueError(f"An error occurred while rendering the template: {reason}") |
| 1001 | wrapper.__cause__ = reason |
| 1002 | assert _template_error_reason(wrapper) == str(reason) |
| 1003 | assert _template_error_reason(reason) == str(reason) |
| 1004 | |
| 1005 | |
| 1006 | def test_template_error_reason_falls_back_to_message(): |
| 1007 | err = ValueError("plain failure") |
| 1008 | assert _template_error_reason(err) == "plain failure" |
| 1009 | |
| 1010 | |
| 1011 | class TestConsolidateSystemMessages: |
| 1012 | def test_no_system_messages_unchanged(self): |
| 1013 | conversation = [ |
| 1014 | {"role": "user", "content": "Hello"}, |
| 1015 | {"role": "assistant", "content": "Hi"}, |
| 1016 | ] |
| 1017 | result = _consolidate_system_messages(conversation) |
| 1018 | assert result == conversation |
| 1019 | |
| 1020 | def test_single_system_at_start_unchanged(self): |
| 1021 | conversation = [ |
| 1022 | {"role": "system", "content": "You are helpful."}, |
| 1023 | {"role": "user", "content": "Hello"}, |
| 1024 | ] |
| 1025 | result = _consolidate_system_messages(conversation) |
| 1026 | assert result == conversation |
| 1027 | |
| 1028 | def test_system_at_non_first_position_moved(self): |
| 1029 | conversation = [ |
| 1030 | {"role": "user", "content": "Hello"}, |
| 1031 | {"role": "system", "content": "You are helpful."}, |
| 1032 | ] |
| 1033 | result = _consolidate_system_messages(conversation) |
| 1034 | assert result[0]["role"] == "system" |
| 1035 | assert result[0]["content"] == "You are helpful." |
| 1036 | assert result[1]["role"] == "user" |
| 1037 | assert result[1]["content"] == "Hello" |
| 1038 | |
| 1039 | def test_multiple_system_messages_merged(self): |
| 1040 | conversation = [ |
| 1041 | {"role": "system", "content": "You are helpful."}, |
| 1042 | {"role": "user", "content": "Hello"}, |
| 1043 | {"role": "system", "content": "Be concise."}, |
| 1044 | ] |
| 1045 | result = _consolidate_system_messages(conversation) |
| 1046 | assert len(result) == 2 |
| 1047 | assert result[0]["role"] == "system" |
| 1048 | assert result[0]["content"] == "You are helpful.\n\nBe concise." |
| 1049 | assert result[1]["role"] == "user" |
| 1050 | |
| 1051 | def test_list_content_handled(self): |
| 1052 | conversation = [ |
| 1053 | {"role": "user", "content": "Hello"}, |
| 1054 | { |
| 1055 | "role": "system", |
| 1056 | "content": [ |
| 1057 | {"type": "text", "text": "Rule 1."}, |
| 1058 | {"type": "text", "text": "Rule 2."}, |
| 1059 | ], |
| 1060 | }, |
| 1061 | ] |
| 1062 | result = _consolidate_system_messages(conversation) |
| 1063 | assert result[0]["role"] == "system" |
| 1064 | assert result[0]["content"] == "Rule 1.\nRule 2." |
| 1065 | assert result[1]["role"] == "user" |
| 1066 | |
| 1067 | def test_does_not_mutate_original(self): |
| 1068 | conversation = [ |
| 1069 | {"role": "user", "content": "Hello"}, |
| 1070 | {"role": "system", "content": "You are helpful."}, |
| 1071 | ] |
| 1072 | original_len = len(conversation) |
| 1073 | _consolidate_system_messages(conversation) |
| 1074 | assert len(conversation) == original_len |
| 1075 | assert conversation[0]["role"] == "user" |
| 1076 | assert conversation[1]["role"] == "system" |