A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 00:15 UTC
[Frontend] Move warmup into Renderer (#36482)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Mar 9, 2026 at 21:03 UTC
3ec2115015334e26b00bb2b4cadc2587138c5948
4 files changed
+60
-158
vllm/entrypoints/openai/chat_completion/serving.py
+8
-37
@@ -72,6 +72,7 @@ from vllm.logprobs import Logprob
72
from vllm.outputs import CompletionOutput, RequestOutput
73
from vllm.parser import ParserManager
74
from vllm.reasoning import ReasoningParser
75
+from vllm.renderers import ChatParams
76
from vllm.sampling_params import BeamSearchParams, SamplingParams
77
from vllm.tokenizers import TokenizerLike
78
from vllm.tool_parsers import ToolParser
@@ -171,44 +172,14 @@ class OpenAIServingChat(OpenAIServing):
172
self.supports_code_interpreter = False
173
self.python_tool = None
174
174
- async def warmup(self) -> None:
175
- """
176
- Warm up the chat template processing to avoid first-request latency.
177
-
178
- This method triggers Jinja2 template compilation and content format
179
- detection that would otherwise happen on the first real request,
180
- causing increased latency on the first request.
181
- """
182
- logger.info("Warming up chat template processing...")
183
- start_time = time.perf_counter()
184
-
185
- try:
186
- # Create a minimal dummy request
187
- dummy_request = ChatCompletionRequest(
188
- messages=[{"role": "user", "content": "warmup"}],
189
- model=None,
190
- max_completion_tokens=1,
175
+ def warmup(self) -> None:
176
+ self.renderer.warmup(
177
+ ChatParams(
178
+ chat_template=self.chat_template,
179
+ chat_template_content_format=self.chat_template_content_format,
180
+ chat_template_kwargs=self.default_chat_template_kwargs,
181
)
192
-
193
- # Call _preprocess_chat to trigger template compilation
194
- # This forces:
195
- # 1. Chat template content format detection
196
- # 2. Jinja2 template compilation
197
- # 3. Tokenizer initialization for chat
198
- await self._preprocess_chat(
199
- dummy_request,
200
- dummy_request.messages,
201
- default_template=self.chat_template,
202
- default_template_content_format=self.chat_template_content_format,
203
- default_template_kwargs=self.default_chat_template_kwargs,
204
- )
205
-
206
- elapsed = (time.perf_counter() - start_time) * 1000
207
- logger.info("Chat template warmup completed in %.1fms", elapsed)
208
-
209
- except Exception:
210
- # Log but don't fail server startup if warmup fails
211
- logger.exception("Chat template warmup failed")
182
+ )
183
184
async def render_chat_request(
185
self,
vllm/entrypoints/openai/generate/api_router.py
+1
-2
@@ -114,9 +114,8 @@ async def init_generate_state(
114
if "generate" in supported_tasks
115
else None
116
)
117
- # Warm up chat template processing to avoid first-request latency
117
if state.openai_serving_chat is not None:
119
- await state.openai_serving_chat.warmup()
118
+ state.openai_serving_chat.warmup()
119
state.openai_serving_completion = (
120
OpenAIServingCompletion(
121
engine_client,
vllm/entrypoints/openai/speech_to_text/speech_to_text.py
+1
-119
@@ -42,10 +42,7 @@ from vllm.exceptions import VLLMValidationError
42
from vllm.inputs import EncoderDecoderInputs, ProcessorInputs
43
from vllm.logger import init_logger
44
from vllm.logprobs import FlatLogprobs, Logprob
45
-from vllm.model_executor.models import (
46
- SupportsTranscription,
47
- supports_transcription,
48
-)
45
+from vllm.model_executor.models import SupportsTranscription
46
from vllm.multimodal.audio import split_audio
47
from vllm.outputs import RequestOutput
48
from vllm.renderers.inputs import DictPrompt, EncoderDecoderDictPrompt
@@ -132,121 +129,6 @@ class OpenAISpeechToText(OpenAIServing):
129
self.default_sampling_params,
130
)
131
135
- # Warm up audio preprocessing to avoid first-request latency
136
- self._warmup_audio_preprocessing()
137
- # Warm up input processor with dummy audio
138
- self._warmup_input_processor()
139
-
140
- def _warmup_audio_preprocessing(self) -> None:
141
- """Warm up audio processing libraries to avoid first-request latency.
142
-
143
- The first call to librosa functions (load, get_duration, mel-spectrogram)
144
- triggers JIT compilation and library initialization which can take ~7s.
145
- This method warms up these operations during server initialization.
146
- """
147
- # Skip warmup if librosa is not installed (optional dependency)
148
- if isinstance(librosa, PlaceholderModule):
149
- return
150
-
151
- # Skip warmup if model doesn't support transcription
152
- if not supports_transcription(self.model_cls):
153
- return
154
-
155
- if getattr(self.model_cls, "skip_warmup_audio_preprocessing", False):
156
- return
157
-
158
- try:
159
- warmup_start = time.perf_counter()
160
- logger.info("Warming up audio preprocessing libraries...")
161
-
162
- # Create a minimal dummy audio (1 second of silence at target sample rate)
163
- dummy_audio = np.zeros(int(self.asr_config.sample_rate), dtype=np.float32)
164
-
165
- # Warm up librosa.load by using librosa functions on the dummy data
166
- # This initializes FFTW, numba JIT, and other audio processing libraries
167
- _ = librosa.get_duration(y=dummy_audio, sr=self.asr_config.sample_rate)
168
-
169
- # Warm up mel-spectrogram computation with model-specific parameters
170
- from vllm.transformers_utils.processor import cached_processor_from_config
171
-
172
- processor = cached_processor_from_config(self.model_config)
173
- feature_extractor = None
174
- if hasattr(processor, "feature_extractor"):
175
- feature_extractor = processor.feature_extractor
176
- elif hasattr(processor, "audio_processor"):
177
- # For models like GraniteSpeech that use audio_processor
178
- audio_proc = processor.audio_processor
179
- if hasattr(audio_proc, "feature_extractor"):
180
- feature_extractor = audio_proc.feature_extractor
181
- # If audio_processor doesn't have feature_extractor,
182
- # skip mel-spectrogram warmup for these models
183
-
184
- if feature_extractor is not None:
185
- _ = librosa.feature.melspectrogram(
186
- y=dummy_audio,
187
- sr=self.asr_config.sample_rate,
188
- n_mels=getattr(feature_extractor, "n_mels", 128),
189
- n_fft=getattr(feature_extractor, "n_fft", 400),
190
- hop_length=getattr(feature_extractor, "hop_length", 160),
191
- )
192
-
193
- warmup_elapsed = time.perf_counter() - warmup_start
194
- logger.info("Audio preprocessing warmup completed in %.2fs", warmup_elapsed)
195
- except Exception:
196
- # Don't fail initialization if warmup fails - log exception and continue
197
- logger.exception(
198
- "Audio preprocessing warmup failed (non-fatal): %s. "
199
- "First request may experience higher latency.",
200
- )
201
-
202
- def _warmup_input_processor(self) -> None:
203
- """Warm up input processor with dummy audio to avoid first-request latency.
204
-
205
- The first call to renderer.render_cmpl() with multimodal audio
206
- triggers multimodal processing initialization which can take ~2.5s.
207
- This method processes a dummy audio request to warm up the pipeline.
208
- """
209
- # Skip warmup if model doesn't support transcription
210
- if not supports_transcription(self.model_cls):
211
- return
212
-
213
- # Only warm up if model supports transcription methods
214
- if not hasattr(self.model_cls, "get_generation_prompt"):
215
- return
216
-
217
- try:
218
- warmup_start = time.perf_counter()
219
- logger.info("Warming up multimodal input processor...")
220
-
221
- # Create minimal dummy audio (1 second of silence)
222
- dummy_audio = np.zeros(int(self.asr_config.sample_rate), dtype=np.float32)
223
-
224
- # Use the same method that _preprocess_speech_to_text uses
225
- # to create the prompt
226
- dummy_prompt = self.model_cls.get_generation_prompt(
227
- audio=dummy_audio,
228
- stt_config=self.asr_config,
229
- model_config=self.model_config,
230
- language="en",
231
- task_type=self.task_type,
232
- request_prompt="",
233
- to_language=None,
234
- )
235
- parsed_prompt = parse_model_prompt(self.model_config, dummy_prompt)
236
-
237
- # Process the dummy input through the input processor
238
- # This will trigger all the multimodal processing initialization
239
- _ = self.renderer.render_cmpl([parsed_prompt])
240
-
241
- warmup_elapsed = time.perf_counter() - warmup_start
242
- logger.info("Input processor warmup completed in %.2fs", warmup_elapsed)
243
- except Exception:
244
- # Don't fail initialization if warmup fails - log warning and continue
245
- logger.exception(
246
- "Input processor warmup failed (non-fatal): %s. "
247
- "First request may experience higher latency."
248
- )
249
-
132
@cached_property
133
def model_cls(self) -> type[SupportsTranscription]:
134
from vllm.model_executor.model_loader import get_model_cls
vllm/renderers/base.py
+50
@@ -158,6 +158,56 @@ class BaseRenderer(ABC, Generic[_T]):
158
if self._mm_cache_stats is not None:
159
self._mm_cache_stats.reset = True
160
161
+ def warmup(self, chat_params: ChatParams) -> None:
162
+ """
163
+ Warm up this renderer to avoid first-request latency.
164
+
165
+ For chat requests:
166
+ - Jinja2 template compilation
167
+
168
+ For multi-modal requests:
169
+ - Importing libraries such as librosa triggers JIT compilation.
170
+ """
171
+ try:
172
+ logger.info("Warming up chat template processing...")
173
+ start_time = time.perf_counter()
174
+
175
+ self.render_chat([[{"role": "user", "content": "warmup"}]], chat_params)
176
+
177
+ elapsed = time.perf_counter() - start_time
178
+ logger.info("Chat template warmup completed in %.3fs", elapsed)
179
+ except Exception:
180
+ logger.exception("Chat template warmup failed")
181
+
182
+ if self.mm_processor:
183
+ from vllm.multimodal.processing import TimingContext
184
+
185
+ model_config = self.model_config
186
+ mm_config = model_config.get_multimodal_config()
187
+ processor = self.mm_processor
188
+ mm_limits = processor.info.allowed_mm_limits
189
+
190
+ try:
191
+ logger.info("Warming up multi-modal processing...")
192
+ start_time = time.perf_counter()
193
+
194
+ processor_inputs = processor.dummy_inputs.get_dummy_processor_inputs(
195
+ seq_len=model_config.max_model_len,
196
+ mm_counts=dict.fromkeys(mm_limits, 1),
197
+ mm_options=mm_config.limit_per_prompt,
198
+ )
199
+ _ = processor.apply(
200
+ processor_inputs,
201
+ timing_ctx=TimingContext(enabled=False),
202
+ )
203
+
204
+ elapsed = time.perf_counter() - start_time
205
+ logger.info("Multi-modal warmup completed in %.3fs", elapsed)
206
+ except Exception:
207
+ logger.exception("Multi-modal warmup failed")
208
+ finally:
209
+ self.clear_mm_cache()
210
+
211
def shutdown(self) -> None:
212
mm_processor_cache = self.mm_processor_cache
213
if mm_processor_cache is not None: