A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 2, 01:00 UTC
[Frontend][Responses API] Move developer-to-system conversion into HF renderer (#43590)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com> Co-authored-by: kdcyberdude <kdsingh.cyberdude@gmail.com> Co-authored-by: Ben Browning <bbrownin@redhat.com>
Chauncey committed
Jun 3, 2026 at 22:52 UTC
27f1d34a23e79fbcf8c988839fed6989ab2645dc
2 files changed
+391
-1
tests/renderers/test_hf.py
+322
@@ -7,6 +7,9 @@ from vllm.config import ModelConfig
7
from vllm.entrypoints.chat_utils import load_chat_template
8
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
9
from vllm.renderers.hf import (
10
+ _consolidate_system_messages,
11
+ _convert_developer_to_system,
12
+ _detect_developer_role_support,
13
_get_hf_base_chat_template_params,
14
_try_extract_ast,
15
resolve_chat_template,
@@ -592,3 +595,322 @@ def test_get_gen_prompt(
595
f"The generated prompt does not match the expected output for "
596
f"model {model} and template {template}"
597
)
598
+
599
+
600
+class TestConvertDeveloperToSystem:
601
+ def test_converts_role(self):
602
+ conversation = [
603
+ {"role": "developer", "content": "You are helpful."},
604
+ {"role": "user", "content": "Hello"},
605
+ ]
606
+ result = _convert_developer_to_system(conversation)
607
+ assert result[0]["role"] == "system"
608
+ assert result[0]["content"] == "You are helpful."
609
+ assert result[1]["role"] == "user"
610
+
611
+ def test_removes_tools_key(self):
612
+ conversation = [
613
+ {
614
+ "role": "developer",
615
+ "content": "Instructions",
616
+ "tools": [{"type": "function"}],
617
+ },
618
+ ]
619
+ result = _convert_developer_to_system(conversation)
620
+ assert "tools" not in result[0]
621
+
622
+ def test_no_developer_messages_unchanged(self):
623
+ conversation = [
624
+ {"role": "system", "content": "System prompt"},
625
+ {"role": "user", "content": "Hello"},
626
+ ]
627
+ result = _convert_developer_to_system(conversation)
628
+ assert result[0]["role"] == "system"
629
+ assert result[1]["role"] == "user"
630
+
631
+ def test_does_not_mutate_original(self):
632
+ original = {
633
+ "role": "developer",
634
+ "content": "Instructions",
635
+ "tools": [{"type": "function"}],
636
+ }
637
+ conversation = [original]
638
+ _convert_developer_to_system(conversation)
639
+ assert original["role"] == "developer"
640
+ assert "tools" in original
641
+
642
+
643
+# --- Developer role detection and conversion tests ---
644
+
645
+CHATML_TEMPLATE = (
646
+ "{% for message in messages %}"
647
+ "{{'<|im_start|>' + message['role'] + '\\n' + message['content']}}"
648
+ "{% if (loop.last and add_generation_prompt) or not loop.last %}"
649
+ "{{ '<|im_end|>' + '\\n'}}"
650
+ "{% endif %}"
651
+ "{% endfor %}"
652
+ "{% if add_generation_prompt and messages[-1]['role'] != 'assistant' %}"
653
+ "{{ '<|im_start|>assistant\\n' }}"
654
+ "{% endif %}"
655
+)
656
+
657
+TEMPLATE_WITH_DEVELOPER = (
658
+ "{% for message in messages %}"
659
+ "{% if message['role'] == 'developer' %}"
660
+ "{{'<|im_start|>developer\\n' + message['content'] + '<|im_end|>\\n'}}"
661
+ "{% elif message['role'] == 'system' %}"
662
+ "{{'<|im_start|>system\\n' + message['content'] + '<|im_end|>\\n'}}"
663
+ "{% elif message['role'] == 'user' %}"
664
+ "{{'<|im_start|>user\\n' + message['content'] + '<|im_end|>\\n'}}"
665
+ "{% elif message['role'] == 'assistant' %}"
666
+ "{{'<|im_start|>assistant\\n' + message['content'] + '<|im_end|>\\n'}}"
667
+ "{% endif %}"
668
+ "{% endfor %}"
669
+ "{% if add_generation_prompt %}"
670
+ "{{ '<|im_start|>assistant\\n' }}"
671
+ "{% endif %}"
672
+)
673
+
674
+STRICT_ROLE_TEMPLATE = (
675
+ "{% for message in messages %}"
676
+ "{% if message['role'] == 'system' %}"
677
+ "{{'<|im_start|>system\\n' + message['content'] + '<|im_end|>\\n'}}"
678
+ "{% elif message['role'] == 'user' %}"
679
+ "{{'<|im_start|>user\\n' + message['content'] + '<|im_end|>\\n'}}"
680
+ "{% elif message['role'] == 'assistant' %}"
681
+ "{{'<|im_start|>assistant\\n' + message['content'] + '<|im_end|>\\n'}}"
682
+ "{% else %}"
683
+ "{{ raise_exception('Unexpected message role: ' + message['role']) }}"
684
+ "{% endif %}"
685
+ "{% endfor %}"
686
+ "{% if add_generation_prompt %}"
687
+ "{{ '<|im_start|>assistant\\n' }}"
688
+ "{% endif %}"
689
+)
690
+
691
+
692
+class TestDetectDeveloperRoleSupport:
693
+ def test_absent_in_chatml(self):
694
+ assert _detect_developer_role_support(CHATML_TEMPLATE) is False
695
+
696
+ def test_present_double_quotes(self):
697
+ assert _detect_developer_role_support(TEMPLATE_WITH_DEVELOPER) is True
698
+
699
+ def test_present_single_quotes(self):
700
+ template = TEMPLATE_WITH_DEVELOPER.replace('"developer"', "'developer'")
701
+ assert _detect_developer_role_support(template) is True
702
+
703
+ def test_absent_in_strict_template(self):
704
+ assert _detect_developer_role_support(STRICT_ROLE_TEMPLATE) is False
705
+
706
+
707
+class TestSafeApplyChatTemplateDeveloperRole:
708
+ @pytest.fixture
709
+ def model_config(self):
710
+ return ModelConfig(
711
+ "facebook/opt-125m",
712
+ tokenizer="facebook/opt-125m",
713
+ tokenizer_mode="auto",
714
+ trust_remote_code=False,
715
+ dtype="float16",
716
+ )
717
+
718
+ @pytest.fixture
719
+ def tokenizer(self):
720
+ return get_tokenizer("facebook/opt-125m")
721
+
722
+ def test_developer_converted_to_system_for_chatml(self, model_config, tokenizer):
723
+ conversation = [
724
+ {"role": "developer", "content": "You are a helpful assistant."},
725
+ {"role": "user", "content": "Hello"},
726
+ ]
727
+ result = safe_apply_chat_template(
728
+ model_config,
729
+ tokenizer,
730
+ conversation,
731
+ chat_template=CHATML_TEMPLATE,
732
+ tokenize=False,
733
+ add_generation_prompt=True,
734
+ )
735
+ assert "<|im_start|>system" in result
736
+ assert "You are a helpful assistant." in result
737
+ assert "<|im_start|>developer" not in result
738
+
739
+ def test_developer_preserved_when_template_supports_it(
740
+ self, model_config, tokenizer
741
+ ):
742
+ conversation = [
743
+ {"role": "developer", "content": "You are a helpful assistant."},
744
+ {"role": "user", "content": "Hello"},
745
+ ]
746
+ result = safe_apply_chat_template(
747
+ model_config,
748
+ tokenizer,
749
+ conversation,
750
+ chat_template=TEMPLATE_WITH_DEVELOPER,
751
+ tokenize=False,
752
+ add_generation_prompt=True,
753
+ )
754
+ assert "<|im_start|>developer" in result
755
+ assert "You are a helpful assistant." in result
756
+
757
+ def test_developer_does_not_crash_strict_template(self, model_config, tokenizer):
758
+ conversation = [
759
+ {"role": "developer", "content": "You are a helpful assistant."},
760
+ {"role": "user", "content": "Hello"},
761
+ ]
762
+ result = safe_apply_chat_template(
763
+ model_config,
764
+ tokenizer,
765
+ conversation,
766
+ chat_template=STRICT_ROLE_TEMPLATE,
767
+ tokenize=False,
768
+ add_generation_prompt=True,
769
+ )
770
+ assert "<|im_start|>system" in result
771
+ assert "You are a helpful assistant." in result
772
+
773
+ def test_no_developer_messages_no_overhead(self, model_config, tokenizer):
774
+ conversation = [
775
+ {"role": "system", "content": "You are helpful."},
776
+ {"role": "user", "content": "Hello"},
777
+ ]
778
+ result = safe_apply_chat_template(
779
+ model_config,
780
+ tokenizer,
781
+ conversation,
782
+ chat_template=CHATML_TEMPLATE,
783
+ tokenize=False,
784
+ add_generation_prompt=True,
785
+ )
786
+ assert "<|im_start|>system" in result
787
+ assert "You are helpful." in result
788
+
789
+ def test_developer_at_non_first_position_consolidated(
790
+ self, model_config, tokenizer
791
+ ):
792
+ conversation = [
793
+ {"role": "system", "content": "You are helpful."},
794
+ {"role": "user", "content": "Hello"},
795
+ {"role": "assistant", "content": "Hi there!"},
796
+ {"role": "developer", "content": "Be concise."},
797
+ {"role": "user", "content": "What is 2+2?"},
798
+ ]
799
+ result = safe_apply_chat_template(
800
+ model_config,
801
+ tokenizer,
802
+ conversation,
803
+ chat_template=SYSTEM_FIRST_TEMPLATE,
804
+ tokenize=False,
805
+ add_generation_prompt=True,
806
+ )
807
+ assert "<|im_start|>system" in result
808
+ assert "You are helpful." in result
809
+ assert "Be concise." in result
810
+ assert "What is 2+2?" in result
811
+
812
+ def test_developer_only_no_prior_system(self, model_config, tokenizer):
813
+ conversation = [
814
+ {"role": "user", "content": "Hello"},
815
+ {"role": "developer", "content": "Be concise."},
816
+ {"role": "user", "content": "What is 2+2?"},
817
+ ]
818
+ result = safe_apply_chat_template(
819
+ model_config,
820
+ tokenizer,
821
+ conversation,
822
+ chat_template=SYSTEM_FIRST_TEMPLATE,
823
+ tokenize=False,
824
+ add_generation_prompt=True,
825
+ )
826
+ assert "<|im_start|>system" in result
827
+ assert "Be concise." in result
828
+
829
+
830
+SYSTEM_FIRST_TEMPLATE = (
831
+ "{% for message in messages %}"
832
+ "{% if message['role'] == 'system' %}"
833
+ "{% if not loop.first %}"
834
+ "{{ raise_exception('System message must be at the beginning.') }}"
835
+ "{% endif %}"
836
+ "{{'<|im_start|>system\\n' + message['content'] + '<|im_end|>\\n'}}"
837
+ "{% elif message['role'] == 'user' %}"
838
+ "{{'<|im_start|>user\\n' + message['content'] + '<|im_end|>\\n'}}"
839
+ "{% elif message['role'] == 'assistant' %}"
840
+ "{{'<|im_start|>assistant\\n' + message['content'] + '<|im_end|>\\n'}}"
841
+ "{% else %}"
842
+ "{{ raise_exception('Unexpected message role: ' + message['role']) }}"
843
+ "{% endif %}"
844
+ "{% endfor %}"
845
+ "{% if add_generation_prompt %}"
846
+ "{{ '<|im_start|>assistant\\n' }}"
847
+ "{% endif %}"
848
+)
849
+
850
+
851
+class TestConsolidateSystemMessages:
852
+ def test_no_system_messages_unchanged(self):
853
+ conversation = [
854
+ {"role": "user", "content": "Hello"},
855
+ {"role": "assistant", "content": "Hi"},
856
+ ]
857
+ result = _consolidate_system_messages(conversation)
858
+ assert result == conversation
859
+
860
+ def test_single_system_at_start_unchanged(self):
861
+ conversation = [
862
+ {"role": "system", "content": "You are helpful."},
863
+ {"role": "user", "content": "Hello"},
864
+ ]
865
+ result = _consolidate_system_messages(conversation)
866
+ assert result == conversation
867
+
868
+ def test_system_at_non_first_position_moved(self):
869
+ conversation = [
870
+ {"role": "user", "content": "Hello"},
871
+ {"role": "system", "content": "You are helpful."},
872
+ ]
873
+ result = _consolidate_system_messages(conversation)
874
+ assert result[0]["role"] == "system"
875
+ assert result[0]["content"] == "You are helpful."
876
+ assert result[1]["role"] == "user"
877
+ assert result[1]["content"] == "Hello"
878
+
879
+ def test_multiple_system_messages_merged(self):
880
+ conversation = [
881
+ {"role": "system", "content": "You are helpful."},
882
+ {"role": "user", "content": "Hello"},
883
+ {"role": "system", "content": "Be concise."},
884
+ ]
885
+ result = _consolidate_system_messages(conversation)
886
+ assert len(result) == 2
887
+ assert result[0]["role"] == "system"
888
+ assert result[0]["content"] == "You are helpful.\n\nBe concise."
889
+ assert result[1]["role"] == "user"
890
+
891
+ def test_list_content_handled(self):
892
+ conversation = [
893
+ {"role": "user", "content": "Hello"},
894
+ {
895
+ "role": "system",
896
+ "content": [
897
+ {"type": "text", "text": "Rule 1."},
898
+ {"type": "text", "text": "Rule 2."},
899
+ ],
900
+ },
901
+ ]
902
+ result = _consolidate_system_messages(conversation)
903
+ assert result[0]["role"] == "system"
904
+ assert result[0]["content"] == "Rule 1.\nRule 2."
905
+ assert result[1]["role"] == "user"
906
+
907
+ def test_does_not_mutate_original(self):
908
+ conversation = [
909
+ {"role": "user", "content": "Hello"},
910
+ {"role": "system", "content": "You are helpful."},
911
+ ]
912
+ original_len = len(conversation)
913
+ _consolidate_system_messages(conversation)
914
+ assert len(conversation) == original_len
915
+ assert conversation[0]["role"] == "user"
916
+ assert conversation[1]["role"] == "system"
vllm/renderers/hf.py
+69
-1
@@ -450,6 +450,66 @@ def _detect_content_format(
450
return "openai"
451
452
453
+@lru_cache(maxsize=32)
454
+def _detect_developer_role_support(chat_template: str) -> bool:
455
+ return '"developer"' in chat_template or "'developer'" in chat_template
456
+
457
+
458
+def _convert_developer_to_system(
459
+ conversation: list[ConversationMessage],
460
+) -> list[ConversationMessage]:
461
+ converted: list[ConversationMessage] = []
462
+ for msg in conversation:
463
+ if msg["role"] == "developer":
464
+ new_msg = dict(msg)
465
+ new_msg["role"] = "system"
466
+ new_msg.pop("tools", None)
467
+ converted.append(new_msg) # type: ignore[arg-type]
468
+ else:
469
+ converted.append(msg)
470
+ return converted
471
+
472
+
473
+def _consolidate_system_messages(
474
+ conversation: list[ConversationMessage],
475
+) -> list[ConversationMessage]:
476
+ """Merge all system messages into one at position 0.
477
+
478
+ Some chat templates (e.g. Qwen 3.6) require the system message to be the
479
+ very first message. After developer-to-system conversion, system messages
480
+ may appear at non-first positions; this merges them into a single message.
481
+ """
482
+ system_contents: list[str] = []
483
+ non_system: list[ConversationMessage] = []
484
+ needs_consolidation = False
485
+ for i, msg in enumerate(conversation):
486
+ if msg["role"] == "system":
487
+ if i > 0 or system_contents:
488
+ needs_consolidation = True
489
+ content = msg.get("content", "")
490
+ if isinstance(content, list):
491
+ parts = []
492
+ for part in content:
493
+ if isinstance(part, dict) and "text" in part:
494
+ parts.append(part["text"])
495
+ elif isinstance(part, str):
496
+ parts.append(part)
497
+ content = "\n".join(parts)
498
+ if content:
499
+ system_contents.append(content)
500
+ else:
501
+ non_system.append(msg)
502
+
503
+ if not needs_consolidation:
504
+ return conversation
505
+
506
+ merged: ConversationMessage = {
507
+ "role": "system",
508
+ "content": "\n\n".join(system_contents),
509
+ }
510
+ return [merged, *non_system]
511
+
512
+
513
def _resolve_chat_template_content_format(
514
chat_template: str | None,
515
tools: list[dict[str, Any]] | None,
@@ -653,7 +713,15 @@ def safe_apply_chat_template(
713
"allowed, so you must provide a chat template if the tokenizer "
714
"does not define one."
715
)
656
-
716
+ if any(
717
+ msg["role"] == "developer" for msg in conversation
718
+ ) and not _detect_developer_role_support(chat_template):
719
+ conversation = _convert_developer_to_system(conversation)
720
+ conversation = _consolidate_system_messages(conversation)
721
+ logger.info_once(
722
+ "Chat template does not support the 'developer' message role. "
723
+ "Converting developer messages to 'system' role.",
724
+ )
725
resolved_kwargs = resolve_chat_template_kwargs(
726
tokenizer=tokenizer,
727
chat_template=chat_template,