A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 07:30 UTC
feat(api): Eager chat template warmup to eliminate first-request latency (#30700)
Signed-off-by: Nathan Price <nathan@abridge.com>
Nathan Price committed
Dec 17, 2025 at 18:01 UTC
05a83dc6ee84be55fef73d5fa6a77fb56d2dd80f
2 files changed
+52
vllm/entrypoints/openai/api_server.py
+3
@@ -1082,6 +1082,9 @@ async def init_app_state(
1082
if "generate" in supported_tasks
1083
else None
1084
)
1085
+ # Warm up chat template processing to avoid first-request latency
1086
+ if state.openai_serving_chat is not None:
1087
+ await state.openai_serving_chat.warmup()
1088
state.openai_serving_completion = (
1089
OpenAIServingCompletion(
1090
engine_client,
vllm/entrypoints/openai/serving_chat.py
+49
@@ -162,6 +162,55 @@ class OpenAIServingChat(OpenAIServing):
162
self.supports_code_interpreter = False
163
self.python_tool = None
164
165
+ async def warmup(self) -> None:
166
+ """
167
+ Warm up the chat template processing to avoid first-request latency.
168
+
169
+ This method triggers Jinja2 template compilation and content format
170
+ detection that would otherwise happen on the first real request,
171
+ causing increased latency on the first request.
172
+ """
173
+ logger.info("Warming up chat template processing...")
174
+ start_time = time.perf_counter()
175
+
176
+ try:
177
+ # Get the tokenizer from the engine
178
+ tokenizer = await self.engine_client.get_tokenizer()
179
+
180
+ # Create a minimal dummy request
181
+ dummy_request = ChatCompletionRequest(
182
+ messages=[{"role": "user", "content": "warmup"}],
183
+ model=None,
184
+ max_completion_tokens=1,
185
+ )
186
+
187
+ # Call _preprocess_chat to trigger template compilation
188
+ # This forces:
189
+ # 1. Chat template content format detection
190
+ # 2. Jinja2 template compilation
191
+ # 3. Tokenizer initialization for chat
192
+ await self._preprocess_chat(
193
+ dummy_request,
194
+ tokenizer,
195
+ dummy_request.messages,
196
+ chat_template=self.chat_template,
197
+ chat_template_content_format=self.chat_template_content_format,
198
+ add_generation_prompt=True,
199
+ continue_final_message=False,
200
+ tool_dicts=None,
201
+ documents=None,
202
+ chat_template_kwargs=None,
203
+ tool_parser=None,
204
+ add_special_tokens=False,
205
+ )
206
+
207
+ elapsed = (time.perf_counter() - start_time) * 1000
208
+ logger.info("Chat template warmup completed in %.1fms", elapsed)
209
+
210
+ except Exception:
211
+ # Log but don't fail server startup if warmup fails
212
+ logger.exception("Chat template warmup failed")
213
+
214
async def create_chat_completion(
215
self,
216
request: ChatCompletionRequest,