[Bugfix] Revert renderer warmup overlap to avoid fork deadlock (#54023)
Signed-off-by: Kevin Luu <51931015+khluu@users.noreply.github.com>
Kevin H. Luu committed
Aug 27, 2026 at 16:36 UTC
d04d3e0a076b7411b6e8ff4b36b4ffb23665179b
4 files changed
+24
-345
tests/renderers/test_warmup.py
-195
@@ -6,18 +6,11 @@ These tests exercise:
6
- Zero-limit modalities are filtered from mm_counts passed to
7
get_dummy_processor_inputs (e.g. --limit-mm-per-prompt image=0 ...)
8
- MM warmup is skipped entirely when mm_processor is None
9
- - The multimodal warmup is launched as a task on the single-worker
10
- ``_mm_executor`` to overlap engine-core init (future lifecycle, join by
11
- warmup/reset/shutdown, no double-run). Routing it through the same
12
- executor that serves ``_process_multimodal`` keeps the numba workqueue
13
- parallel region single-threaded, avoiding the "Concurrent access has been
14
- detected" fatal abort when a request arrives during warmup.
9
10
No model weights are required: warmup() is called directly on a MagicMock
11
that acts as the renderer instance.
12
"""
13
20
-from concurrent.futures import Future, ThreadPoolExecutor
14
from unittest.mock import MagicMock, patch
15
16
from vllm.renderers.base import BaseRenderer
@@ -46,18 +39,6 @@ def _make_renderer_mock(mm_limits: dict[str, int]) -> MagicMock:
39
renderer, BaseRenderer
40
)
41
renderer._clear_processor_cache = BaseRenderer._clear_processor_cache
49
- renderer.warmup_mm = BaseRenderer.warmup_mm.__get__(renderer, BaseRenderer)
50
- renderer.start_mm_warmup_in_background = (
51
- BaseRenderer.start_mm_warmup_in_background.__get__(renderer, BaseRenderer)
52
- )
53
- renderer._join_mm_warmup = BaseRenderer._join_mm_warmup.__get__(
54
- renderer, BaseRenderer
55
- )
56
- renderer.shutdown = BaseRenderer.shutdown.__get__(renderer, BaseRenderer)
57
- # No background warmup launched by default; warmup() takes the inline path.
58
- renderer._mm_warmup_future = None
59
- # MM warmup has not run yet; warmup_mm must actually execute on the mock.
60
- renderer._mm_warmup_done = False
42
renderer.clear_mm_cache = MagicMock()
43
renderer.model_config.max_model_len = 128
44
renderer.model_config.get_multimodal_config.return_value.limit_per_prompt = {}
@@ -152,179 +133,3 @@ class TestReadonlyMmWarmup:
133
134
readonly_mm_processor.apply.assert_called_once()
135
readonly_mm_processor.cache.clear_cache.assert_called_once()
155
-
156
-
157
-class TestChatWarmupDispatched:
158
- """The chat-template warmup task must always run."""
159
-
160
- def test_render_chat_called_with_warmup_message(self):
161
- renderer = _make_renderer_mock({"image": 1})
162
-
163
- with patch("vllm.multimodal.processing.TimingContext", autospec=True):
164
- BaseRenderer.warmup(renderer, ChatParams())
165
-
166
- renderer.render_chat.assert_called_once()
167
-
168
-
169
-class TestWarmupFaultIsolation:
170
- """A failure during a multimodal processor warmup is caught so it does not
171
- abort the remaining warmup steps; warmup itself must not raise."""
172
-
173
- def test_chat_failure_does_not_abort_mm_warmup(self):
174
- renderer = _make_renderer_mock({"image": 1})
175
- renderer.render_chat.side_effect = RuntimeError("chat boom")
176
-
177
- with patch("vllm.multimodal.processing.TimingContext", autospec=True):
178
- BaseRenderer.warmup(renderer, ChatParams()) # must not raise
179
-
180
- renderer.mm_processor.apply.assert_called_once()
181
-
182
- def test_mm_failure_does_not_abort_readonly_warmup(self):
183
- renderer = _make_renderer_mock({"image": 1})
184
- readonly_mm_processor = MagicMock()
185
- readonly_mm_processor.info.allowed_mm_limits = {"image": 1}
186
- renderer._readonly_mm_processor = readonly_mm_processor
187
- # main processor warmup blows up before apply()
188
- renderer.mm_processor.dummy_inputs.get_dummy_processor_inputs.side_effect = (
189
- RuntimeError("mm boom")
190
- )
191
-
192
- with patch("vllm.multimodal.processing.TimingContext", autospec=True):
193
- BaseRenderer.warmup(renderer, ChatParams()) # must not raise
194
-
195
- readonly_mm_processor.apply.assert_called_once()
196
- # cache is still cleared in the failed task's finally
197
- renderer.clear_mm_cache.assert_called_once()
198
-
199
-
200
-class TestBackgroundMmWarmup:
201
- """The multimodal warmup is launched as a task on the single-worker
202
- ``_mm_executor`` so it overlaps engine-core init (fork) while staying
203
- serialized with the serving path (``_process_multimodal`` runs on the same
204
- executor). warmup()/reset_mm_cache/shutdown join it. This also fixes the
205
- numba workqueue "Concurrent access has been detected" abort: warmup and a
206
- concurrent serving request can no longer both enter the numba parallel
207
- region at once."""
208
-
209
- def _make_bg_renderer(self, mm_limits: dict[str, int]):
210
- """A renderer mock with a real single-worker executor so submit()
211
- returns a real Future and warmup_mm actually runs (in the worker)."""
212
- renderer = _make_renderer_mock(mm_limits)
213
- renderer._mm_executor = ThreadPoolExecutor(max_workers=1)
214
- return renderer
215
-
216
- def test_start_submits_future_to_mm_executor(self):
217
- renderer = self._make_bg_renderer({"image": 1})
218
- try:
219
- with patch("vllm.multimodal.processing.TimingContext", autospec=True):
220
- # Spy on the real executor's submit to confirm the warmup is
221
- # dispatched through _mm_executor (not a separate Thread).
222
- with patch.object(
223
- renderer._mm_executor, "submit", wraps=renderer._mm_executor.submit
224
- ) as spy:
225
- renderer.start_mm_warmup_in_background()
226
- assert isinstance(renderer._mm_warmup_future, Future)
227
- assert spy.called
228
- renderer._join_mm_warmup()
229
- assert renderer._mm_warmup_future is None
230
- renderer.mm_processor.apply.assert_called_once()
231
- finally:
232
- renderer._mm_executor.shutdown(wait=True)
233
-
234
- def test_start_noop_for_text_only_model(self):
235
- renderer = _make_renderer_mock({})
236
- renderer.mm_processor = None
237
- # _readonly_mm_processor is already None
238
-
239
- renderer.start_mm_warmup_in_background()
240
-
241
- assert renderer._mm_warmup_future is None
242
-
243
- def test_start_is_run_at_most_once(self):
244
- renderer = self._make_bg_renderer({"image": 1})
245
- try:
246
- with patch("vllm.multimodal.processing.TimingContext", autospec=True):
247
- renderer.start_mm_warmup_in_background()
248
- first = renderer._mm_warmup_future
249
- renderer.start_mm_warmup_in_background() # must not spawn a second
250
- assert renderer._mm_warmup_future is first
251
- renderer._join_mm_warmup()
252
- finally:
253
- renderer._mm_executor.shutdown(wait=True)
254
-
255
- def test_warmup_joins_background_and_does_not_rerun_mm(self):
256
- """When a background MM warmup is in flight, warmup() must join it and
257
- run only the chat warmup — the MM warmup must not run twice."""
258
- renderer = self._make_bg_renderer({"image": 1})
259
- try:
260
- with patch("vllm.multimodal.processing.TimingContext", autospec=True):
261
- renderer.start_mm_warmup_in_background()
262
- BaseRenderer.warmup(renderer, ChatParams())
263
-
264
- # MM apply called exactly once (by the background warmup task).
265
- renderer.mm_processor.apply.assert_called_once()
266
- # Chat warmup ran exactly once.
267
- renderer.render_chat.assert_called_once()
268
- # Background task has been joined and cleared.
269
- assert renderer._mm_warmup_future is None
270
- finally:
271
- renderer._mm_executor.shutdown(wait=True)
272
-
273
- def test_warmup_does_not_rerun_mm_after_reset_joins_background(self):
274
- """Regression: reset_mm_cache joins the background warmup before
275
- warmup() runs (it clears _mm_warmup_future). warmup() must still not
276
- re-run the MM warmup — the _mm_warmup_done flag survives the join."""
277
- renderer = self._make_bg_renderer({"image": 1})
278
- try:
279
- with patch("vllm.multimodal.processing.TimingContext", autospec=True):
280
- renderer.start_mm_warmup_in_background()
281
- # Simulate reset_mm_cache joining the background warmup first
282
- # (clears _mm_warmup_future, sets _mm_warmup_done via the join).
283
- renderer._join_mm_warmup()
284
- assert renderer._mm_warmup_future is None # joined & cleared
285
- BaseRenderer.warmup(renderer, ChatParams())
286
-
287
- # MM apply called exactly once (by the background warmup task);
288
- # warmup() did not re-run warmup_mm despite the future being cleared.
289
- renderer.mm_processor.apply.assert_called_once()
290
- renderer.render_chat.assert_called_once()
291
- finally:
292
- renderer._mm_executor.shutdown(wait=True)
293
-
294
- def test_warmup_mm_runs_at_most_once(self):
295
- """Direct repeated calls to warmup_mm run the MM warmup only once."""
296
- renderer = _make_renderer_mock({"image": 1})
297
-
298
- with patch("vllm.multimodal.processing.TimingContext", autospec=True):
299
- renderer.warmup_mm()
300
- renderer.warmup_mm() # second call must be a no-op
301
-
302
- renderer.mm_processor.apply.assert_called_once()
303
-
304
- def test_join_mm_warmup_blocks_until_apply_done(self):
305
- renderer = self._make_bg_renderer({"image": 1})
306
- try:
307
- with patch("vllm.multimodal.processing.TimingContext", autospec=True):
308
- renderer.start_mm_warmup_in_background()
309
- renderer._join_mm_warmup()
310
-
311
- assert renderer._mm_warmup_future is None
312
- renderer.mm_processor.apply.assert_called_once()
313
- finally:
314
- renderer._mm_executor.shutdown(wait=True)
315
-
316
- def test_shutdown_joins_background_warmup(self):
317
- """shutdown() must join the background warmup before closing caches,
318
- so the mm_processor_cache is never touched concurrently."""
319
- renderer = self._make_bg_renderer({"image": 1})
320
-
321
- with patch("vllm.multimodal.processing.TimingContext", autospec=True):
322
- renderer.start_mm_warmup_in_background()
323
- future = renderer._mm_warmup_future
324
- renderer.shutdown()
325
-
326
- assert renderer._mm_warmup_future is None
327
- # The background warmup was allowed to complete (apply ran) and the
328
- # future is done after shutdown joined it.
329
- assert future.done()
330
- renderer.mm_processor.apply.assert_called_once()
vllm/renderers/base.py
+24
-116
@@ -4,7 +4,7 @@ import asyncio
4
import time
5
from abc import ABC, abstractmethod
6
from collections.abc import Mapping, Sequence
7
-from concurrent.futures import Executor, Future, ThreadPoolExecutor
7
+from concurrent.futures import Executor, ThreadPoolExecutor
8
from functools import cached_property
9
from typing import TYPE_CHECKING, Any, Generic, overload
10
@@ -79,26 +79,6 @@ class BaseRenderer(ABC, Generic[_T]):
79
80
self.tokenizer = tokenizer
81
82
- # Background multimodal warmup task (a future returned by
83
- # ``_mm_executor``) so it overlaps engine-core initialization while
84
- # staying serialized with the serving path through the same
85
- # single-worker executor. None when no background warmup has been
86
- # launched (or it has already been joined). Routing the warmup through
87
- # ``_mm_executor`` (max_workers=1) guarantees it can never run
88
- # concurrently with a serving request inside the numba workqueue
89
- # parallel region, which would otherwise hit numba's
90
- # "Concurrent access has been detected" fatal abort (see
91
- # vllm/utils/jit_monitor.py, which forces the workqueue layer).
92
- # Joined by the first of reset_mm_cache / warmup / shutdown so the
93
- # mm_processor_cache is also never touched concurrently by the warmup
94
- # and the serving path.
95
- self._mm_warmup_future: Future[None] | None = None
96
- # Whether warmup_mm has already run. Survives _join_mm_warmup (which
97
- # clears the thread pointer), so warmup() can tell that the background
98
- # warmup already ran even after reset_mm_cache joined it first, and
99
- # avoid running warmup_mm a second time.
100
- self._mm_warmup_done: bool = False
101
-
82
# Thread pool executor for blocking tokenizer operations. The
83
# multimodal processor receives a deep-copied tokenizer (see #36557)
84
# so it is safe to run tokenization and MM preprocessing concurrently.
@@ -254,97 +234,12 @@ class BaseRenderer(ABC, Generic[_T]):
234
elapsed = time.perf_counter() - start_time
235
logger.info("%s warmup completed in %.3fs", log_prefix, elapsed)
236
257
- def warmup_mm(self) -> None:
258
- # Run only the multimodal warmup (mm + readonly mm), serially.
259
- # Idempotent: runs at most once. A background warmup
260
- # (start_mm_warmup_in_background) may already have run it, and
261
- # reset_mm_cache may have joined that background warmup task before
262
- # warmup() reaches this point — the _mm_warmup_done flag survives the
263
- # join (the future pointer does not), so warmup() can still tell it
264
- # already ran.
265
- if self._mm_warmup_done:
266
- return
267
- try:
268
- # prevent MM processor hangs
269
- with set_default_torch_num_threads(1):
270
- if self.mm_processor:
271
- try:
272
- logger.debug("Warming up multi-modal processing...")
273
- self._warmup_mm_processor(
274
- self.mm_processor,
275
- log_prefix="Multi-modal",
276
- )
277
- except Exception:
278
- logger.warning("Multi-modal warmup failed")
279
- finally:
280
- self.clear_mm_cache()
281
-
282
- if self._readonly_mm_processor is not None:
283
- try:
284
- logger.debug("Warming up readonly multi-modal processing...")
285
- self._warmup_mm_processor(
286
- self._readonly_mm_processor,
287
- log_prefix="Readonly multi-modal",
288
- )
289
- except Exception:
290
- logger.warning("Readonly multi-modal warmup failed")
291
- finally:
292
- self._clear_processor_cache(self._readonly_mm_processor)
293
- finally:
294
- self._mm_warmup_done = True
295
-
296
- def start_mm_warmup_in_background(self) -> None:
297
- # Launch the multimodal warmup as a task on the single-worker
298
- # ``_mm_executor`` so it overlaps with engine-core initialization
299
- # (model loading) while staying serialized with the serving path.
300
- #
301
- # Why the *same* executor as serving (not a separate Thread):
302
- # ``_process_multimodal`` runs on ``_mm_executor`` (max_workers=1) and
303
- # enters the numba workqueue parallel region (e.g. Kimi K2.5 vision
304
- # preprocessing via kimi_k25_vision_fused). A separate warmup Thread
305
- # would run concurrently with that single serving worker, so both
306
- # threads would enter the numba workqueue parallel region at the same
307
- # time and trip numba's "Concurrent access has been detected" ->
308
- # "Fatal Python error: Aborted" (see vllm/utils/jit_monitor.py, which
309
- # forces the non-threadsafe workqueue layer). Submitting the warmup to
310
- # the *same* single-worker executor makes warmup and serving
311
- # physically incapable of overlapping on the numba threading layer,
312
- # while still overlapping at the process level with the (forked)
313
- # engine-core init in the main thread.
314
- #
315
- # The warmup task is joined by the first of: reset_mm_cache, warmup, or
316
- # shutdown. This also guarantees the frontend mm_processor_cache —
317
- # which is not designed for concurrent access (its get_and_update must
318
- # stay ordered with the engine-core cache) — is never touched
319
- # concurrently by the warmup and the serving path.
320
- if self.mm_processor is None and self._readonly_mm_processor is None:
321
- return # text-only model: nothing to warm up
322
- if self._mm_warmup_future is not None:
323
- return # already launched
324
- self._mm_warmup_future = self._mm_executor.submit(self.warmup_mm)
325
-
326
- def _join_mm_warmup(self) -> None:
327
- # Wait for the background MM warmup task to finish, if one was started.
328
- # Called from the main / event-loop thread (reset_mm_cache, warmup,
329
- # shutdown) — never from inside the ``_mm_executor`` worker — so
330
- # waiting on the future cannot deadlock.
331
- future = getattr(self, "_mm_warmup_future", None)
332
- if future is not None:
333
- future.result()
334
- self._mm_warmup_future = None
335
-
237
def warmup(self, chat_params: ChatParams) -> None:
238
"""
239
Warm up this renderer to avoid first-request latency.
240
241
For chat requests:
242
- Jinja2 template compilation
342
-
343
- If a background multimodal warmup was launched (to overlap
344
- engine-core init) it is joined first so it is not still touching the
345
- mm_processor_cache. warmup_mm is idempotent, so the call below is a
346
- no-op if the background warmup — or a prior reset_mm_cache that joined
347
- it — already ran it.
243
"""
244
from vllm.entrypoints.chat_utils import ChatTemplateResolutionError
245
@@ -363,12 +258,29 @@ class BaseRenderer(ABC, Generic[_T]):
258
except Exception:
259
logger.warning("Chat template warmup failed", exc_info=True)
260
366
- if self._mm_warmup_future is not None:
367
- # Join a background MM warmup launched to overlap engine-core init.
368
- self._join_mm_warmup()
369
- # Idempotent: no-op if the background warmup (or a prior reset_mm_cache)
370
- # already ran the MM warmup.
371
- self.warmup_mm()
261
+ if self.mm_processor:
262
+ try:
263
+ logger.debug("Warming up multi-modal processing...")
264
+ self._warmup_mm_processor(
265
+ self.mm_processor,
266
+ log_prefix="Multi-modal",
267
+ )
268
+ except Exception:
269
+ logger.warning("Multi-modal warmup failed")
270
+ finally:
271
+ self.clear_mm_cache()
272
+
273
+ if self._readonly_mm_processor is not None:
274
+ try:
275
+ logger.debug("Warming up readonly multi-modal processing...")
276
+ self._warmup_mm_processor(
277
+ self._readonly_mm_processor,
278
+ log_prefix="Readonly multi-modal",
279
+ )
280
+ except Exception:
281
+ logger.warning("Readonly multi-modal warmup failed")
282
+ finally:
283
+ self._clear_processor_cache(self._readonly_mm_processor)
284
285
async def clear_mm_cache_async(self) -> None:
286
"""Serialize clear_mm_cache through the multimodal executor to avoid
@@ -376,10 +288,6 @@ class BaseRenderer(ABC, Generic[_T]):
288
await self._clear_mm_cache_async()
289
290
def shutdown(self) -> None:
379
- # Wait for any background MM warmup to finish before closing the
380
- # mm_processor_cache and stopping the executors.
381
- self._join_mm_warmup()
382
-
291
mm_processor_cache = self.mm_processor_cache
292
if mm_processor_cache is not None:
293
mm_processor_cache.close()
vllm/v1/engine/async_llm.py
-17
@@ -137,19 +137,6 @@ class AsyncLLM(EngineClient):
137
# Convert EngineInput --> EngineCoreRequest.
138
self.input_processor = InputProcessor(self.vllm_config, renderer)
139
140
- # Launch the multimodal warmup in the background so it overlaps
141
- # engine-core initialization (model loading, started below). Must be
142
- # after InputProcessor: its MultiModalBudget runs a synchronous MM
143
- # processing pass (get_dummy_mm_inputs -> processor.apply), and the HF
144
- # processor's Numba parallel kernels (e.g. Kimi-K2.5 vision fused) are
145
- # not thread-safe — launching the warmup thread any earlier would run
146
- # it concurrently with that pass and abort. The MM warmup is
147
- # frontend-only (sends no engine_core requests) and only needs the
148
- # renderer. Joined by reset_mm_cache / warmup / shutdown so the
149
- # mm_processor_cache is never touched concurrently by the warmup and
150
- # the serving path.
151
- renderer.start_mm_warmup_in_background()
152
-
140
# Converts EngineCoreOutputs --> RequestOutput.
141
self.output_processor = OutputProcessor(
142
renderer.tokenizer,
@@ -969,10 +956,6 @@ class AsyncLLM(EngineClient):
956
await asyncio.gather(*coros)
957
958
async def reset_mm_cache(self) -> None:
972
- # Join any background MM warmup before clearing: the mm_processor_cache
973
- # is not designed for concurrent access, so the warmup's apply/clear
974
- # must have finished before we clear here.
975
- self.renderer._join_mm_warmup()
959
await self.renderer.clear_mm_cache_async()
960
await self.engine_core.reset_mm_cache_async()
961
vllm/v1/engine/llm_engine.py
-17
@@ -93,19 +93,6 @@ class LLMEngine:
93
# Convert EngineInput --> EngineCoreRequest.
94
self.input_processor = InputProcessor(self.vllm_config, renderer)
95
96
- # Launch the multimodal warmup in the background so it overlaps
97
- # engine-core initialization (model loading, started below). Must be
98
- # after InputProcessor: its MultiModalBudget runs a synchronous MM
99
- # processing pass (get_dummy_mm_inputs -> processor.apply), and the HF
100
- # processor's Numba parallel kernels (e.g. Kimi-K2.5 vision fused) are
101
- # not thread-safe — launching the warmup thread any earlier would run
102
- # it concurrently with that pass and abort. The MM warmup is
103
- # frontend-only (sends no engine_core requests) and only needs the
104
- # renderer. Joined by reset_mm_cache / warmup / shutdown so the
105
- # mm_processor_cache is never touched concurrently by the warmup and
106
- # the serving path.
107
- renderer.start_mm_warmup_in_background()
108
-
96
# Converts EngineCoreOutputs --> RequestOutput.
97
self.output_processor = OutputProcessor(
98
renderer.tokenizer,
@@ -356,10 +343,6 @@ class LLMEngine:
343
self.engine_core.profile(False)
344
345
def reset_mm_cache(self):
359
- # Join any background MM warmup before clearing: the mm_processor_cache
360
- # is not designed for concurrent access, so the warmup's apply/clear
361
- # must have finished before we clear here.
362
- self.renderer._join_mm_warmup()
346
self.renderer.clear_mm_cache()
347
self.engine_core.reset_mm_cache()
348