[Bugfix][Frontend] Restore the chat template content format mismatch warning (#54622)
Signed-off-by: Pratham Shirbhate <prathamshirbhate1909@gmail.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
Pratham Shirbhate committed
Sep 1, 2026 at 23:47 UTC
0ad5652a523d7bfe191a08b97ca45ead585b1897
2 files changed
+51
-33
tests/entrypoints/openai/chat_completion/test_serving_chat.py
+2
@@ -539,6 +539,8 @@ class MockModelConfig:
539
trust_remote_code = False
540
tokenizer_mode = "auto"
541
max_model_len = 100
542
+ revision = None
543
+ code_revision = None
544
tokenizer_revision = None
545
multimodal_config = MultiModalConfig()
546
hf_config = MockHFConfig()
vllm/renderers/hf.py
+49
-33
@@ -509,6 +509,22 @@ def _consolidate_system_messages(
509
return [merged, *non_system]
510
511
512
+_CONTENT_FORMATS_MAXSIZE = 32
513
+_CONTENT_FORMATS = dict[
514
+ tuple[str | None, bool, str, str | None, str | None, bool],
515
+ "ChatTemplateContentFormat",
516
+]()
517
+"""
518
+Used in `_resolve_chat_template_content_format` to avoid resolving and parsing
519
+the chat template on every request.
520
+
521
+`lru_cache` cannot be used because `tools` is a list and `ModelConfig` defines
522
+`__eq__` without `__hash__`, so the key holds the fields `resolve_chat_template`
523
+selects the template by. Only the presence of `tools` is part of it: it decides
524
+whether the processor template is tried, and its contents never reach the AST.
525
+"""
526
+
527
+
528
def _resolve_chat_template_content_format(
529
chat_template: str | None,
530
tools: list[dict[str, Any]] | None,
@@ -516,6 +532,17 @@ def _resolve_chat_template_content_format(
532
*,
533
model_config: ModelConfig,
534
) -> ChatTemplateContentFormat:
535
+ cache_key = (
536
+ chat_template,
537
+ tools is None,
538
+ tokenizer.name_or_path,
539
+ model_config.revision,
540
+ model_config.code_revision,
541
+ model_config.trust_remote_code,
542
+ )
543
+ if (cached_format := _CONTENT_FORMATS.get(cache_key)) is not None:
544
+ return cached_format
545
+
546
resolved_chat_template = resolve_chat_template(
547
tokenizer,
548
chat_template=chat_template,
@@ -535,31 +562,12 @@ def _resolve_chat_template_content_format(
562
else _detect_content_format(jinja_text, default="string")
563
)
564
538
- return detected_format
539
-
540
-
541
-@lru_cache
542
-def _log_chat_template_content_format(
543
- chat_template: str | None, # For caching purposes
544
- given_format: ChatTemplateContentFormatOption,
545
- detected_format: ChatTemplateContentFormatOption,
546
-):
547
- logger.info(
548
- "Detected the chat template content format to be '%s'. "
549
- "You can set `--chat-template-content-format` to override this.",
550
- detected_format,
551
- )
565
+ # Requests may carry their own chat template, so bound the cache.
566
+ if len(_CONTENT_FORMATS) >= _CONTENT_FORMATS_MAXSIZE:
567
+ _CONTENT_FORMATS.clear()
568
+ _CONTENT_FORMATS[cache_key] = detected_format
569
553
- if given_format != "auto" and given_format != detected_format:
554
- logger.warning(
555
- "You specified `--chat-template-content-format %s` "
556
- "which is different from the detected format '%s'. "
557
- "If our automatic detection is incorrect, please consider "
558
- "opening a GitHub issue so that we can improve it: "
559
- "https://github.com/vllm-project/vllm/issues/new/choose",
560
- given_format,
561
- detected_format,
562
- )
570
+ return detected_format
571
572
573
def resolve_chat_template_content_format(
@@ -570,9 +578,6 @@ def resolve_chat_template_content_format(
578
*,
579
model_config: ModelConfig,
580
) -> ChatTemplateContentFormat:
573
- if given_format != "auto":
574
- return given_format
575
-
581
detected_format = _resolve_chat_template_content_format(
582
chat_template,
583
tools,
@@ -580,13 +585,24 @@ def resolve_chat_template_content_format(
585
model_config=model_config,
586
)
587
583
- _log_chat_template_content_format(
584
- chat_template,
585
- given_format=given_format,
586
- detected_format=detected_format,
587
- )
588
+ if given_format == "auto":
589
+ logger.info_once(
590
+ "Detected the chat template content format to be '%s'. "
591
+ "You can set `--chat-template-content-format` to override this.",
592
+ detected_format,
593
+ )
594
+ elif given_format != detected_format:
595
+ logger.warning_once(
596
+ "You specified `--chat-template-content-format %s` "
597
+ "which is different from the detected format '%s'. "
598
+ "If our automatic detection is incorrect, please consider "
599
+ "opening a GitHub issue so that we can improve it: "
600
+ "https://github.com/vllm-project/vllm/issues/new/choose",
601
+ given_format,
602
+ detected_format,
603
+ )
604
589
- return detected_format
605
+ return detected_format if given_format == "auto" else given_format
606
607
608
# adapted from https://github.com/huggingface/transformers/blob/v4.56.2/src/transformers/utils/chat_template_utils.py#L398-L412