A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Frontend] Cache chat template kwargs resolution (#26227)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Isotr0py committed
Oct 4, 2025 at 23:32 UTC
a42d2df75f932711deee18e0f4d8ca92fe7ece8c
7 files changed
+81
-18
vllm/entrypoints/chat_utils.py
+17
-7
@@ -1572,6 +1572,22 @@ class AssistantTracker(jinja2.ext.Extension):
1572
return call_block.set_lineno(lineno)
1573
1574
1575
+def _resolve_chat_template_kwargs(
1576
+ chat_template: str,
1577
+):
1578
+ env = jinja2.sandbox.ImmutableSandboxedEnvironment(
1579
+ trim_blocks=True,
1580
+ lstrip_blocks=True,
1581
+ extensions=[AssistantTracker, jinja2.ext.loopcontrols],
1582
+ )
1583
+ parsed_content = env.parse(chat_template)
1584
+ template_vars = jinja2.meta.find_undeclared_variables(parsed_content)
1585
+ return template_vars
1586
+
1587
+
1588
+_cached_resolve_chat_template_kwargs = lru_cache(_resolve_chat_template_kwargs)
1589
+
1590
+
1591
def resolve_chat_template_kwargs(
1592
tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
1593
chat_template: str,
@@ -1582,13 +1598,7 @@ def resolve_chat_template_kwargs(
1598
if supports_kw(tokenizer.apply_chat_template, k, allow_var_kwargs=False)
1599
}
1600
1585
- env = jinja2.sandbox.ImmutableSandboxedEnvironment(
1586
- trim_blocks=True,
1587
- lstrip_blocks=True,
1588
- extensions=[AssistantTracker, jinja2.ext.loopcontrols],
1589
- )
1590
- parsed_content = env.parse(chat_template)
1591
- template_vars = jinja2.meta.find_undeclared_variables(parsed_content)
1601
+ template_vars = _cached_resolve_chat_template_kwargs(chat_template)
1602
1603
# We exclude chat_template from kwargs here, because
1604
# chat template has been already resolved at this stage
vllm/entrypoints/openai/api_server.py
+3
@@ -1745,6 +1745,7 @@ async def init_app_state(
1745
request_logger=request_logger,
1746
chat_template=resolved_chat_template,
1747
chat_template_content_format=args.chat_template_content_format,
1748
+ trust_request_chat_template=args.trust_request_chat_template,
1749
log_error_stack=args.log_error_stack,
1750
) if "encode" in supported_tasks else None
1751
state.openai_serving_embedding = OpenAIServingEmbedding(
@@ -1754,6 +1755,7 @@ async def init_app_state(
1755
request_logger=request_logger,
1756
chat_template=resolved_chat_template,
1757
chat_template_content_format=args.chat_template_content_format,
1758
+ trust_request_chat_template=args.trust_request_chat_template,
1759
log_error_stack=args.log_error_stack,
1760
) if "embed" in supported_tasks else None
1761
state.openai_serving_classification = ServingClassification(
@@ -1777,6 +1779,7 @@ async def init_app_state(
1779
request_logger=request_logger,
1780
chat_template=resolved_chat_template,
1781
chat_template_content_format=args.chat_template_content_format,
1782
+ trust_request_chat_template=args.trust_request_chat_template,
1783
log_error_stack=args.log_error_stack,
1784
)
1785
state.openai_serving_transcription = OpenAIServingTranscription(
vllm/entrypoints/openai/serving_chat.py
+9
-11
@@ -222,16 +222,14 @@ class OpenAIServingChat(OpenAIServing):
222
223
if not self.use_harmony:
224
# Common case.
225
- request_chat_template = request.chat_template
226
- chat_template_kwargs = request.chat_template_kwargs
227
- if not self.trust_request_chat_template and (
228
- request_chat_template is not None or
229
- (chat_template_kwargs and
230
- chat_template_kwargs.get("chat_template") is not None)):
231
- return self.create_error_response(
232
- "Chat template is passed with request, but "
233
- "--trust-request-chat-template is not set. "
234
- "Refused request with untrusted chat template.")
225
+ error_check_ret = self._validate_chat_template(
226
+ request_chat_template=request.chat_template,
227
+ chat_template_kwargs=request.chat_template_kwargs,
228
+ trust_request_chat_template=self.
229
+ trust_request_chat_template,
230
+ )
231
+ if error_check_ret is not None:
232
+ return error_check_ret
233
(
234
conversation,
235
request_prompts,
@@ -240,7 +238,7 @@ class OpenAIServingChat(OpenAIServing):
238
request,
239
tokenizer,
240
request.messages,
243
- chat_template=request_chat_template or self.chat_template,
241
+ chat_template=request.chat_template or self.chat_template,
242
chat_template_content_format=self.
243
chat_template_content_format,
244
add_generation_prompt=request.add_generation_prompt,
vllm/entrypoints/openai/serving_embedding.py
+16
@@ -576,6 +576,7 @@ class OpenAIServingEmbedding(EmbeddingMixin):
576
request_logger: Optional[RequestLogger],
577
chat_template: Optional[str],
578
chat_template_content_format: ChatTemplateContentFormatOption,
579
+ trust_request_chat_template: bool = False,
580
log_error_stack: bool = False,
581
) -> None:
582
super().__init__(engine_client=engine_client,
@@ -586,6 +587,7 @@ class OpenAIServingEmbedding(EmbeddingMixin):
587
588
self.chat_template = chat_template
589
self.chat_template_content_format: Final = chat_template_content_format
590
+ self.trust_request_chat_template = trust_request_chat_template
591
592
async def create_embedding(
593
self,
@@ -629,3 +631,17 @@ class OpenAIServingEmbedding(EmbeddingMixin):
631
return self.create_error_response(str(e))
632
633
return pooling_params
634
+
635
+ async def _preprocess(
636
+ self,
637
+ ctx: ServeContext,
638
+ ) -> Optional[ErrorResponse]:
639
+ if isinstance(ctx.request, EmbeddingChatRequest):
640
+ error_check_ret = self._validate_chat_template(
641
+ request_chat_template=ctx.request.chat_template,
642
+ chat_template_kwargs=ctx.request.chat_template_kwargs,
643
+ trust_request_chat_template=self.trust_request_chat_template,
644
+ )
645
+ if error_check_ret is not None:
646
+ return error_check_ret
647
+ return await super()._preprocess(ctx)
vllm/entrypoints/openai/serving_engine.py
+16
@@ -751,6 +751,22 @@ class OpenAIServing:
751
tokenizer=tokenizer,
752
)
753
754
+ def _validate_chat_template(
755
+ self,
756
+ request_chat_template: Optional[str],
757
+ chat_template_kwargs: Optional[dict[str, Any]],
758
+ trust_request_chat_template: bool,
759
+ ) -> Optional[ErrorResponse]:
760
+ if not trust_request_chat_template and (
761
+ request_chat_template is not None or
762
+ (chat_template_kwargs
763
+ and chat_template_kwargs.get("chat_template") is not None)):
764
+ return self.create_error_response(
765
+ "Chat template is passed with request, but "
766
+ "--trust-request-chat-template is not set. "
767
+ "Refused request with untrusted chat template.")
768
+ return None
769
+
770
async def _preprocess_chat(
771
self,
772
request: Union[ChatLikeRequest, ResponsesRequest],
vllm/entrypoints/openai/serving_pooling.py
+10
@@ -65,6 +65,7 @@ class OpenAIServingPooling(OpenAIServing):
65
request_logger: Optional[RequestLogger],
66
chat_template: Optional[str],
67
chat_template_content_format: ChatTemplateContentFormatOption,
68
+ trust_request_chat_template: bool = False,
69
log_error_stack: bool = False,
70
) -> None:
71
super().__init__(engine_client=engine_client,
@@ -75,6 +76,7 @@ class OpenAIServingPooling(OpenAIServing):
76
77
self.chat_template = chat_template
78
self.chat_template_content_format: Final = chat_template_content_format
79
+ self.trust_request_chat_template = trust_request_chat_template
80
io_processor_plugin = self.model_config.io_processor_plugin
81
self.io_processor = get_io_processor(vllm_config, io_processor_plugin)
82
@@ -129,6 +131,14 @@ class OpenAIServingPooling(OpenAIServing):
131
prompt=validated_prompt, request_id=request_id)
132
133
elif isinstance(request, PoolingChatRequest):
134
+ error_check_ret = self._validate_chat_template(
135
+ request_chat_template=request.chat_template,
136
+ chat_template_kwargs=request.chat_template_kwargs,
137
+ trust_request_chat_template=self.
138
+ trust_request_chat_template,
139
+ )
140
+ if error_check_ret is not None:
141
+ return error_check_ret
142
(
143
_,
144
_,
vllm/entrypoints/openai/serving_tokenization.py
+10
@@ -40,6 +40,7 @@ class OpenAIServingTokenization(OpenAIServing):
40
request_logger: Optional[RequestLogger],
41
chat_template: Optional[str],
42
chat_template_content_format: ChatTemplateContentFormatOption,
43
+ trust_request_chat_template: bool = False,
44
log_error_stack: bool = False,
45
) -> None:
46
super().__init__(engine_client=engine_client,
@@ -50,6 +51,7 @@ class OpenAIServingTokenization(OpenAIServing):
51
52
self.chat_template = chat_template
53
self.chat_template_content_format: Final = chat_template_content_format
54
+ self.trust_request_chat_template = trust_request_chat_template
55
56
async def create_tokenize(
57
self,
@@ -71,6 +73,14 @@ class OpenAIServingTokenization(OpenAIServing):
73
if isinstance(request, TokenizeChatRequest):
74
tool_dicts = (None if request.tools is None else
75
[tool.model_dump() for tool in request.tools])
76
+ error_check_ret = self._validate_chat_template(
77
+ request_chat_template=request.chat_template,
78
+ chat_template_kwargs=request.chat_template_kwargs,
79
+ trust_request_chat_template=self.
80
+ trust_request_chat_template,
81
+ )
82
+ if error_check_ret is not None:
83
+ return error_check_ret
84
(
85
_,
86
_,