[warmup] overlap renderer warmup and engine core initialization (#52764)
Signed-off-by: Andy Xie <andy.xning@gmail.com>
Ning Xie committed
Aug 27, 2026 at 11:42 UTC
ca90b9e7d4e3ec670143e4b1822bb856ab0260cc
4 files changed
+345
-24
tests/renderers/test_warmup.py
+195
@@ -6,11 +6,18 @@ These tests exercise:
6
- Zero-limit modalities are filtered from mm_counts passed to
7
get_dummy_processor_inputs (e.g. --limit-mm-per-prompt image=0 ...)
8
- MM warmup is skipped entirely when mm_processor is None
9
+ - The multimodal warmup is launched as a task on the single-worker
10
+ ``_mm_executor`` to overlap engine-core init (future lifecycle, join by
11
+ warmup/reset/shutdown, no double-run). Routing it through the same
12
+ executor that serves ``_process_multimodal`` keeps the numba workqueue
13
+ parallel region single-threaded, avoiding the "Concurrent access has been
14
+ detected" fatal abort when a request arrives during warmup.
15
16
No model weights are required: warmup() is called directly on a MagicMock
17
that acts as the renderer instance.
18
"""
19
20
+from concurrent.futures import Future, ThreadPoolExecutor
21
from unittest.mock import MagicMock, patch
22
23
from vllm.renderers.base import BaseRenderer
@@ -39,6 +46,18 @@ def _make_renderer_mock(mm_limits: dict[str, int]) -> MagicMock:
46
renderer, BaseRenderer
47
)
48
renderer._clear_processor_cache = BaseRenderer._clear_processor_cache
49
+ renderer.warmup_mm = BaseRenderer.warmup_mm.__get__(renderer, BaseRenderer)
50
+ renderer.start_mm_warmup_in_background = (
51
+ BaseRenderer.start_mm_warmup_in_background.__get__(renderer, BaseRenderer)
52
+ )
53
+ renderer._join_mm_warmup = BaseRenderer._join_mm_warmup.__get__(
54
+ renderer, BaseRenderer
55
+ )
56
+ renderer.shutdown = BaseRenderer.shutdown.__get__(renderer, BaseRenderer)
57
+ # No background warmup launched by default; warmup() takes the inline path.
58
+ renderer._mm_warmup_future = None
59
+ # MM warmup has not run yet; warmup_mm must actually execute on the mock.
60
+ renderer._mm_warmup_done = False
61
renderer.clear_mm_cache = MagicMock()
62
renderer.model_config.max_model_len = 128
63
renderer.model_config.get_multimodal_config.return_value.limit_per_prompt = {}
@@ -133,3 +152,179 @@ class TestReadonlyMmWarmup:
152
153
readonly_mm_processor.apply.assert_called_once()
154
readonly_mm_processor.cache.clear_cache.assert_called_once()
155
+
156
+
157
+class TestChatWarmupDispatched:
158
+ """The chat-template warmup task must always run."""
159
+
160
+ def test_render_chat_called_with_warmup_message(self):
161
+ renderer = _make_renderer_mock({"image": 1})
162
+
163
+ with patch("vllm.multimodal.processing.TimingContext", autospec=True):
164
+ BaseRenderer.warmup(renderer, ChatParams())
165
+
166
+ renderer.render_chat.assert_called_once()
167
+
168
+
169
+class TestWarmupFaultIsolation:
170
+ """A failure during a multimodal processor warmup is caught so it does not
171
+ abort the remaining warmup steps; warmup itself must not raise."""
172
+
173
+ def test_chat_failure_does_not_abort_mm_warmup(self):
174
+ renderer = _make_renderer_mock({"image": 1})
175
+ renderer.render_chat.side_effect = RuntimeError("chat boom")
176
+
177
+ with patch("vllm.multimodal.processing.TimingContext", autospec=True):
178
+ BaseRenderer.warmup(renderer, ChatParams()) # must not raise
179
+
180
+ renderer.mm_processor.apply.assert_called_once()
181
+
182
+ def test_mm_failure_does_not_abort_readonly_warmup(self):
183
+ renderer = _make_renderer_mock({"image": 1})
184
+ readonly_mm_processor = MagicMock()
185
+ readonly_mm_processor.info.allowed_mm_limits = {"image": 1}
186
+ renderer._readonly_mm_processor = readonly_mm_processor
187
+ # main processor warmup blows up before apply()
188
+ renderer.mm_processor.dummy_inputs.get_dummy_processor_inputs.side_effect = (
189
+ RuntimeError("mm boom")
190
+ )
191
+
192
+ with patch("vllm.multimodal.processing.TimingContext", autospec=True):
193
+ BaseRenderer.warmup(renderer, ChatParams()) # must not raise
194
+
195
+ readonly_mm_processor.apply.assert_called_once()
196
+ # cache is still cleared in the failed task's finally
197
+ renderer.clear_mm_cache.assert_called_once()
198
+
199
+
200
+class TestBackgroundMmWarmup:
201
+ """The multimodal warmup is launched as a task on the single-worker
202
+ ``_mm_executor`` so it overlaps engine-core init (fork) while staying
203
+ serialized with the serving path (``_process_multimodal`` runs on the same
204
+ executor). warmup()/reset_mm_cache/shutdown join it. This also fixes the
205
+ numba workqueue "Concurrent access has been detected" abort: warmup and a
206
+ concurrent serving request can no longer both enter the numba parallel
207
+ region at once."""
208
+
209
+ def _make_bg_renderer(self, mm_limits: dict[str, int]):
210
+ """A renderer mock with a real single-worker executor so submit()
211
+ returns a real Future and warmup_mm actually runs (in the worker)."""
212
+ renderer = _make_renderer_mock(mm_limits)
213
+ renderer._mm_executor = ThreadPoolExecutor(max_workers=1)
214
+ return renderer
215
+
216
+ def test_start_submits_future_to_mm_executor(self):
217
+ renderer = self._make_bg_renderer({"image": 1})
218
+ try:
219
+ with patch("vllm.multimodal.processing.TimingContext", autospec=True):
220
+ # Spy on the real executor's submit to confirm the warmup is
221
+ # dispatched through _mm_executor (not a separate Thread).
222
+ with patch.object(
223
+ renderer._mm_executor, "submit", wraps=renderer._mm_executor.submit
224
+ ) as spy:
225
+ renderer.start_mm_warmup_in_background()
226
+ assert isinstance(renderer._mm_warmup_future, Future)
227
+ assert spy.called
228
+ renderer._join_mm_warmup()
229
+ assert renderer._mm_warmup_future is None
230
+ renderer.mm_processor.apply.assert_called_once()
231
+ finally:
232
+ renderer._mm_executor.shutdown(wait=True)
233
+
234
+ def test_start_noop_for_text_only_model(self):
235
+ renderer = _make_renderer_mock({})
236
+ renderer.mm_processor = None
237
+ # _readonly_mm_processor is already None
238
+
239
+ renderer.start_mm_warmup_in_background()
240
+
241
+ assert renderer._mm_warmup_future is None
242
+
243
+ def test_start_is_run_at_most_once(self):
244
+ renderer = self._make_bg_renderer({"image": 1})
245
+ try:
246
+ with patch("vllm.multimodal.processing.TimingContext", autospec=True):
247
+ renderer.start_mm_warmup_in_background()
248
+ first = renderer._mm_warmup_future
249
+ renderer.start_mm_warmup_in_background() # must not spawn a second
250
+ assert renderer._mm_warmup_future is first
251
+ renderer._join_mm_warmup()
252
+ finally:
253
+ renderer._mm_executor.shutdown(wait=True)
254
+
255
+ def test_warmup_joins_background_and_does_not_rerun_mm(self):
256
+ """When a background MM warmup is in flight, warmup() must join it and
257
+ run only the chat warmup — the MM warmup must not run twice."""
258
+ renderer = self._make_bg_renderer({"image": 1})
259
+ try:
260
+ with patch("vllm.multimodal.processing.TimingContext", autospec=True):
261
+ renderer.start_mm_warmup_in_background()
262
+ BaseRenderer.warmup(renderer, ChatParams())
263
+
264
+ # MM apply called exactly once (by the background warmup task).
265
+ renderer.mm_processor.apply.assert_called_once()
266
+ # Chat warmup ran exactly once.
267
+ renderer.render_chat.assert_called_once()
268
+ # Background task has been joined and cleared.
269
+ assert renderer._mm_warmup_future is None
270
+ finally:
271
+ renderer._mm_executor.shutdown(wait=True)
272
+
273
+ def test_warmup_does_not_rerun_mm_after_reset_joins_background(self):
274
+ """Regression: reset_mm_cache joins the background warmup before
275
+ warmup() runs (it clears _mm_warmup_future). warmup() must still not
276
+ re-run the MM warmup — the _mm_warmup_done flag survives the join."""
277
+ renderer = self._make_bg_renderer({"image": 1})
278
+ try:
279
+ with patch("vllm.multimodal.processing.TimingContext", autospec=True):
280
+ renderer.start_mm_warmup_in_background()
281
+ # Simulate reset_mm_cache joining the background warmup first
282
+ # (clears _mm_warmup_future, sets _mm_warmup_done via the join).
283
+ renderer._join_mm_warmup()
284
+ assert renderer._mm_warmup_future is None # joined & cleared
285
+ BaseRenderer.warmup(renderer, ChatParams())
286
+
287
+ # MM apply called exactly once (by the background warmup task);
288
+ # warmup() did not re-run warmup_mm despite the future being cleared.
289
+ renderer.mm_processor.apply.assert_called_once()
290
+ renderer.render_chat.assert_called_once()
291
+ finally:
292
+ renderer._mm_executor.shutdown(wait=True)
293
+
294
+ def test_warmup_mm_runs_at_most_once(self):
295
+ """Direct repeated calls to warmup_mm run the MM warmup only once."""
296
+ renderer = _make_renderer_mock({"image": 1})
297
+
298
+ with patch("vllm.multimodal.processing.TimingContext", autospec=True):
299
+ renderer.warmup_mm()
300
+ renderer.warmup_mm() # second call must be a no-op
301
+
302
+ renderer.mm_processor.apply.assert_called_once()
303
+
304
+ def test_join_mm_warmup_blocks_until_apply_done(self):
305
+ renderer = self._make_bg_renderer({"image": 1})
306
+ try:
307
+ with patch("vllm.multimodal.processing.TimingContext", autospec=True):
308
+ renderer.start_mm_warmup_in_background()
309
+ renderer._join_mm_warmup()
310
+
311
+ assert renderer._mm_warmup_future is None
312
+ renderer.mm_processor.apply.assert_called_once()
313
+ finally:
314
+ renderer._mm_executor.shutdown(wait=True)
315
+
316
+ def test_shutdown_joins_background_warmup(self):
317
+ """shutdown() must join the background warmup before closing caches,
318
+ so the mm_processor_cache is never touched concurrently."""
319
+ renderer = self._make_bg_renderer({"image": 1})
320
+
321
+ with patch("vllm.multimodal.processing.TimingContext", autospec=True):
322
+ renderer.start_mm_warmup_in_background()
323
+ future = renderer._mm_warmup_future
324
+ renderer.shutdown()
325
+
326
+ assert renderer._mm_warmup_future is None
327
+ # The background warmup was allowed to complete (apply ran) and the
328
+ # future is done after shutdown joined it.
329
+ assert future.done()
330
+ renderer.mm_processor.apply.assert_called_once()
vllm/renderers/base.py
+116
-24
@@ -4,7 +4,7 @@ import asyncio
4
import time
5
from abc import ABC, abstractmethod
6
from collections.abc import Mapping, Sequence
7
-from concurrent.futures import Executor, ThreadPoolExecutor
7
+from concurrent.futures import Executor, Future, ThreadPoolExecutor
8
from functools import cached_property
9
from typing import TYPE_CHECKING, Any, Generic, overload
10
@@ -79,6 +79,26 @@ class BaseRenderer(ABC, Generic[_T]):
79
80
self.tokenizer = tokenizer
81
82
+ # Background multimodal warmup task (a future returned by
83
+ # ``_mm_executor``) so it overlaps engine-core initialization while
84
+ # staying serialized with the serving path through the same
85
+ # single-worker executor. None when no background warmup has been
86
+ # launched (or it has already been joined). Routing the warmup through
87
+ # ``_mm_executor`` (max_workers=1) guarantees it can never run
88
+ # concurrently with a serving request inside the numba workqueue
89
+ # parallel region, which would otherwise hit numba's
90
+ # "Concurrent access has been detected" fatal abort (see
91
+ # vllm/utils/jit_monitor.py, which forces the workqueue layer).
92
+ # Joined by the first of reset_mm_cache / warmup / shutdown so the
93
+ # mm_processor_cache is also never touched concurrently by the warmup
94
+ # and the serving path.
95
+ self._mm_warmup_future: Future[None] | None = None
96
+ # Whether warmup_mm has already run. Survives _join_mm_warmup (which
97
+ # clears the thread pointer), so warmup() can tell that the background
98
+ # warmup already ran even after reset_mm_cache joined it first, and
99
+ # avoid running warmup_mm a second time.
100
+ self._mm_warmup_done: bool = False
101
+
102
# Thread pool executor for blocking tokenizer operations. The
103
# multimodal processor receives a deep-copied tokenizer (see #36557)
104
# so it is safe to run tokenization and MM preprocessing concurrently.
@@ -234,12 +254,97 @@ class BaseRenderer(ABC, Generic[_T]):
254
elapsed = time.perf_counter() - start_time
255
logger.info("%s warmup completed in %.3fs", log_prefix, elapsed)
256
257
+ def warmup_mm(self) -> None:
258
+ # Run only the multimodal warmup (mm + readonly mm), serially.
259
+ # Idempotent: runs at most once. A background warmup
260
+ # (start_mm_warmup_in_background) may already have run it, and
261
+ # reset_mm_cache may have joined that background warmup task before
262
+ # warmup() reaches this point — the _mm_warmup_done flag survives the
263
+ # join (the future pointer does not), so warmup() can still tell it
264
+ # already ran.
265
+ if self._mm_warmup_done:
266
+ return
267
+ try:
268
+ # prevent MM processor hangs
269
+ with set_default_torch_num_threads(1):
270
+ if self.mm_processor:
271
+ try:
272
+ logger.debug("Warming up multi-modal processing...")
273
+ self._warmup_mm_processor(
274
+ self.mm_processor,
275
+ log_prefix="Multi-modal",
276
+ )
277
+ except Exception:
278
+ logger.warning("Multi-modal warmup failed")
279
+ finally:
280
+ self.clear_mm_cache()
281
+
282
+ if self._readonly_mm_processor is not None:
283
+ try:
284
+ logger.debug("Warming up readonly multi-modal processing...")
285
+ self._warmup_mm_processor(
286
+ self._readonly_mm_processor,
287
+ log_prefix="Readonly multi-modal",
288
+ )
289
+ except Exception:
290
+ logger.warning("Readonly multi-modal warmup failed")
291
+ finally:
292
+ self._clear_processor_cache(self._readonly_mm_processor)
293
+ finally:
294
+ self._mm_warmup_done = True
295
+
296
+ def start_mm_warmup_in_background(self) -> None:
297
+ # Launch the multimodal warmup as a task on the single-worker
298
+ # ``_mm_executor`` so it overlaps with engine-core initialization
299
+ # (model loading) while staying serialized with the serving path.
300
+ #
301
+ # Why the *same* executor as serving (not a separate Thread):
302
+ # ``_process_multimodal`` runs on ``_mm_executor`` (max_workers=1) and
303
+ # enters the numba workqueue parallel region (e.g. Kimi K2.5 vision
304
+ # preprocessing via kimi_k25_vision_fused). A separate warmup Thread
305
+ # would run concurrently with that single serving worker, so both
306
+ # threads would enter the numba workqueue parallel region at the same
307
+ # time and trip numba's "Concurrent access has been detected" ->
308
+ # "Fatal Python error: Aborted" (see vllm/utils/jit_monitor.py, which
309
+ # forces the non-threadsafe workqueue layer). Submitting the warmup to
310
+ # the *same* single-worker executor makes warmup and serving
311
+ # physically incapable of overlapping on the numba threading layer,
312
+ # while still overlapping at the process level with the (forked)
313
+ # engine-core init in the main thread.
314
+ #
315
+ # The warmup task is joined by the first of: reset_mm_cache, warmup, or
316
+ # shutdown. This also guarantees the frontend mm_processor_cache —
317
+ # which is not designed for concurrent access (its get_and_update must
318
+ # stay ordered with the engine-core cache) — is never touched
319
+ # concurrently by the warmup and the serving path.
320
+ if self.mm_processor is None and self._readonly_mm_processor is None:
321
+ return # text-only model: nothing to warm up
322
+ if self._mm_warmup_future is not None:
323
+ return # already launched
324
+ self._mm_warmup_future = self._mm_executor.submit(self.warmup_mm)
325
+
326
+ def _join_mm_warmup(self) -> None:
327
+ # Wait for the background MM warmup task to finish, if one was started.
328
+ # Called from the main / event-loop thread (reset_mm_cache, warmup,
329
+ # shutdown) — never from inside the ``_mm_executor`` worker — so
330
+ # waiting on the future cannot deadlock.
331
+ future = getattr(self, "_mm_warmup_future", None)
332
+ if future is not None:
333
+ future.result()
334
+ self._mm_warmup_future = None
335
+
336
def warmup(self, chat_params: ChatParams) -> None:
337
"""
338
Warm up this renderer to avoid first-request latency.
339
340
For chat requests:
341
- Jinja2 template compilation
342
+
343
+ If a background multimodal warmup was launched (to overlap
344
+ engine-core init) it is joined first so it is not still touching the
345
+ mm_processor_cache. warmup_mm is idempotent, so the call below is a
346
+ no-op if the background warmup — or a prior reset_mm_cache that joined
347
+ it — already ran it.
348
"""
349
from vllm.entrypoints.chat_utils import ChatTemplateResolutionError
350
@@ -258,29 +363,12 @@ class BaseRenderer(ABC, Generic[_T]):
363
except Exception:
364
logger.warning("Chat template warmup failed", exc_info=True)
365
261
- if self.mm_processor:
262
- try:
263
- logger.debug("Warming up multi-modal processing...")
264
- self._warmup_mm_processor(
265
- self.mm_processor,
266
- log_prefix="Multi-modal",
267
- )
268
- except Exception:
269
- logger.warning("Multi-modal warmup failed")
270
- finally:
271
- self.clear_mm_cache()
272
-
273
- if self._readonly_mm_processor is not None:
274
- try:
275
- logger.debug("Warming up readonly multi-modal processing...")
276
- self._warmup_mm_processor(
277
- self._readonly_mm_processor,
278
- log_prefix="Readonly multi-modal",
279
- )
280
- except Exception:
281
- logger.warning("Readonly multi-modal warmup failed")
282
- finally:
283
- self._clear_processor_cache(self._readonly_mm_processor)
366
+ if self._mm_warmup_future is not None:
367
+ # Join a background MM warmup launched to overlap engine-core init.
368
+ self._join_mm_warmup()
369
+ # Idempotent: no-op if the background warmup (or a prior reset_mm_cache)
370
+ # already ran the MM warmup.
371
+ self.warmup_mm()
372
373
async def clear_mm_cache_async(self) -> None:
374
"""Serialize clear_mm_cache through the multimodal executor to avoid
@@ -288,6 +376,10 @@ class BaseRenderer(ABC, Generic[_T]):
376
await self._clear_mm_cache_async()
377
378
def shutdown(self) -> None:
379
+ # Wait for any background MM warmup to finish before closing the
380
+ # mm_processor_cache and stopping the executors.
381
+ self._join_mm_warmup()
382
+
383
mm_processor_cache = self.mm_processor_cache
384
if mm_processor_cache is not None:
385
mm_processor_cache.close()
vllm/v1/engine/async_llm.py
+17
@@ -137,6 +137,19 @@ class AsyncLLM(EngineClient):
137
# Convert EngineInput --> EngineCoreRequest.
138
self.input_processor = InputProcessor(self.vllm_config, renderer)
139
140
+ # Launch the multimodal warmup in the background so it overlaps
141
+ # engine-core initialization (model loading, started below). Must be
142
+ # after InputProcessor: its MultiModalBudget runs a synchronous MM
143
+ # processing pass (get_dummy_mm_inputs -> processor.apply), and the HF
144
+ # processor's Numba parallel kernels (e.g. Kimi-K2.5 vision fused) are
145
+ # not thread-safe — launching the warmup thread any earlier would run
146
+ # it concurrently with that pass and abort. The MM warmup is
147
+ # frontend-only (sends no engine_core requests) and only needs the
148
+ # renderer. Joined by reset_mm_cache / warmup / shutdown so the
149
+ # mm_processor_cache is never touched concurrently by the warmup and
150
+ # the serving path.
151
+ renderer.start_mm_warmup_in_background()
152
+
153
# Converts EngineCoreOutputs --> RequestOutput.
154
self.output_processor = OutputProcessor(
155
renderer.tokenizer,
@@ -956,6 +969,10 @@ class AsyncLLM(EngineClient):
969
await asyncio.gather(*coros)
970
971
async def reset_mm_cache(self) -> None:
972
+ # Join any background MM warmup before clearing: the mm_processor_cache
973
+ # is not designed for concurrent access, so the warmup's apply/clear
974
+ # must have finished before we clear here.
975
+ self.renderer._join_mm_warmup()
976
await self.renderer.clear_mm_cache_async()
977
await self.engine_core.reset_mm_cache_async()
978
vllm/v1/engine/llm_engine.py
+17
@@ -93,6 +93,19 @@ class LLMEngine:
93
# Convert EngineInput --> EngineCoreRequest.
94
self.input_processor = InputProcessor(self.vllm_config, renderer)
95
96
+ # Launch the multimodal warmup in the background so it overlaps
97
+ # engine-core initialization (model loading, started below). Must be
98
+ # after InputProcessor: its MultiModalBudget runs a synchronous MM
99
+ # processing pass (get_dummy_mm_inputs -> processor.apply), and the HF
100
+ # processor's Numba parallel kernels (e.g. Kimi-K2.5 vision fused) are
101
+ # not thread-safe — launching the warmup thread any earlier would run
102
+ # it concurrently with that pass and abort. The MM warmup is
103
+ # frontend-only (sends no engine_core requests) and only needs the
104
+ # renderer. Joined by reset_mm_cache / warmup / shutdown so the
105
+ # mm_processor_cache is never touched concurrently by the warmup and
106
+ # the serving path.
107
+ renderer.start_mm_warmup_in_background()
108
+
109
# Converts EngineCoreOutputs --> RequestOutput.
110
self.output_processor = OutputProcessor(
111
renderer.tokenizer,
@@ -343,6 +356,10 @@ class LLMEngine:
356
self.engine_core.profile(False)
357
358
def reset_mm_cache(self):
359
+ # Join any background MM warmup before clearing: the mm_processor_cache
360
+ # is not designed for concurrent access, so the warmup's apply/clear
361
+ # must have finished before we clear here.
362
+ self.renderer._join_mm_warmup()
363
self.renderer.clear_mm_cache()
364
self.engine_core.reset_mm_cache()
365