A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 06:15 UTC
fix: add warmup for audio preprocessing (#30706)
Signed-off-by: Nathan Price <nathan@abridge.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
Nathan Price committed
Dec 18, 2025 at 00:12 UTC
fc2ae6d6177c0b27a10f6c930e335ac0ec240982
1 file changed
+126
-1
vllm/entrypoints/openai/speech_to_text.py
+126
-1
@@ -35,7 +35,7 @@ from vllm.entrypoints.openai.serving_engine import OpenAIServing, SpeechToTextRe
35
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
36
from vllm.inputs.data import PromptType
37
from vllm.logger import init_logger
38
-from vllm.model_executor.models import SupportsTranscription
38
+from vllm.model_executor.models import SupportsTranscription, supports_transcription
39
from vllm.outputs import RequestOutput
40
from vllm.tokenizers import get_tokenizer
41
from vllm.utils.import_utils import PlaceholderModule
@@ -112,6 +112,131 @@ class OpenAISpeechToText(OpenAIServing):
112
self.default_sampling_params,
113
)
114
115
+ # Warm up audio preprocessing to avoid first-request latency
116
+ self._warmup_audio_preprocessing()
117
+ # Warm up input processor with dummy audio
118
+ self._warmup_input_processor()
119
+
120
+ def _warmup_audio_preprocessing(self) -> None:
121
+ """Warm up audio processing libraries to avoid first-request latency.
122
+
123
+ The first call to librosa functions (load, get_duration, mel-spectrogram)
124
+ triggers JIT compilation and library initialization which can take ~7s.
125
+ This method warms up these operations during server initialization.
126
+ """
127
+ # Skip warmup if librosa is not installed (optional dependency)
128
+ if isinstance(librosa, PlaceholderModule):
129
+ return
130
+
131
+ # Skip warmup if model doesn't support transcription
132
+ if not supports_transcription(self.model_cls):
133
+ return
134
+
135
+ try:
136
+ warmup_start = time.perf_counter()
137
+ logger.info("Warming up audio preprocessing libraries...")
138
+
139
+ # Create a minimal dummy audio (1 second of silence at target sample rate)
140
+ dummy_audio = np.zeros(int(self.asr_config.sample_rate), dtype=np.float32)
141
+
142
+ # Warm up librosa.load by using librosa functions on the dummy data
143
+ # This initializes FFTW, numba JIT, and other audio processing libraries
144
+ _ = librosa.get_duration(y=dummy_audio, sr=self.asr_config.sample_rate)
145
+
146
+ # Warm up mel-spectrogram computation with model-specific parameters
147
+ from vllm.transformers_utils.processor import (
148
+ cached_processor_from_config,
149
+ )
150
+
151
+ processor = cached_processor_from_config(self.model_config)
152
+ feature_extractor = None
153
+ if hasattr(processor, "feature_extractor"):
154
+ feature_extractor = processor.feature_extractor
155
+ elif hasattr(processor, "audio_processor"):
156
+ # For models like GraniteSpeech that use audio_processor
157
+ audio_proc = processor.audio_processor
158
+ if hasattr(audio_proc, "feature_extractor"):
159
+ feature_extractor = audio_proc.feature_extractor
160
+ # If audio_processor doesn't have feature_extractor,
161
+ # skip mel-spectrogram warmup for these models
162
+
163
+ if feature_extractor is not None:
164
+ _ = librosa.feature.melspectrogram(
165
+ y=dummy_audio,
166
+ sr=self.asr_config.sample_rate,
167
+ n_mels=getattr(feature_extractor, "n_mels", 128),
168
+ n_fft=getattr(feature_extractor, "n_fft", 400),
169
+ hop_length=getattr(feature_extractor, "hop_length", 160),
170
+ )
171
+
172
+ warmup_elapsed = time.perf_counter() - warmup_start
173
+ logger.info("Audio preprocessing warmup completed in %.2fs", warmup_elapsed)
174
+ except Exception:
175
+ # Don't fail initialization if warmup fails - log exception and continue
176
+ logger.exception(
177
+ "Audio preprocessing warmup failed (non-fatal): %s. "
178
+ "First request may experience higher latency.",
179
+ )
180
+
181
+ def _warmup_input_processor(self) -> None:
182
+ """Warm up input processor with dummy audio to avoid first-request latency.
183
+
184
+ The first call to input_processor.process_inputs() with multimodal audio
185
+ triggers multimodal processing initialization which can take ~2.5s.
186
+ This method processes a dummy audio request to warm up the pipeline.
187
+ """
188
+ # Skip warmup if model doesn't support transcription
189
+ if not supports_transcription(self.model_cls):
190
+ return
191
+
192
+ # Only warm up if model supports transcription methods
193
+ if not hasattr(self.model_cls, "get_generation_prompt"):
194
+ return
195
+
196
+ try:
197
+ from vllm.sampling_params import SamplingParams
198
+
199
+ warmup_start = time.perf_counter()
200
+ logger.info("Warming up multimodal input processor...")
201
+
202
+ # Create minimal dummy audio (1 second of silence)
203
+ dummy_audio = np.zeros(int(self.asr_config.sample_rate), dtype=np.float32)
204
+
205
+ # Use the same method that _preprocess_speech_to_text uses
206
+ # to create the prompt
207
+ dummy_prompt = self.model_cls.get_generation_prompt(
208
+ audio=dummy_audio,
209
+ stt_config=self.asr_config,
210
+ model_config=self.model_config,
211
+ language="en",
212
+ task_type=self.task_type,
213
+ request_prompt="",
214
+ to_language=None,
215
+ )
216
+
217
+ # Create minimal sampling params
218
+ dummy_params = SamplingParams(
219
+ max_tokens=1,
220
+ temperature=0.0,
221
+ )
222
+
223
+ # Process the dummy input through the input processor
224
+ # This will trigger all the multimodal processing initialization
225
+ _ = self.input_processor.process_inputs(
226
+ request_id="warmup",
227
+ prompt=dummy_prompt,
228
+ params=dummy_params,
229
+ )
230
+
231
+ warmup_elapsed = time.perf_counter() - warmup_start
232
+ logger.info("Input processor warmup completed in %.2fs", warmup_elapsed)
233
+ except Exception:
234
+ # Don't fail initialization if warmup fails - log warning and continue
235
+ logger.exception(
236
+ "Input processor warmup failed (non-fatal): %s. "
237
+ "First request may experience higher latency."
238
+ )
239
+
240
@cached_property
241
def model_cls(self) -> type[SupportsTranscription]:
242
from vllm.model_executor.model_loader import get_model_cls