@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 2a336d8239

[warmup] overlap renderer warmup and engine core initialization (#54557)

Signed-off-by: Andy Xie <andy.xning@gmail.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>

Ning Xie committed Sep 4, 2026 at 03:44 UTC 2a336d8239d00565a99752cfa248f2326e2ac8aa
5 files changed +561 -33
tests/renderers/test_warmup.py
+346 -1
@@ -6,13 +6,25 @@ These tests exercise:
6 - Zero-limit modalities are filtered from mm_counts passed to
7 get_dummy_processor_inputs (e.g. --limit-mm-per-prompt image=0 ...)
8 - MM warmup is skipped entirely when mm_processor is None
9 + - The multimodal warmup is launched as a task on the single-worker
10 + _mm_executor to overlap engine-core init (future lifecycle, join by
11 + warmup/reset/shutdown, no double-run). Routing it through the same
12 + executor that serves _process_multimodal keeps the numba workqueue
13 + parallel region single-threaded, avoiding the "Concurrent access has been
14 + detected" fatal abort when a request arrives during warmup.
15 + - That overlap only exists for the multiprocess clients: the in-process
16 + EngineCore is built synchronously, so InprocClient rejects a renderer
17 + (assert) and warmup stays inside renderer.warmup().
18
19 No model weights are required: warmup() is called directly on a MagicMock
20 that acts as the renderer instance.
21 """
22
23 +from concurrent.futures import Future, ThreadPoolExecutor
24 from unittest.mock import MagicMock, patch
25
26 +import pytest
27 +
28 from vllm.renderers.base import BaseRenderer
29 from vllm.renderers.params import ChatParams
30
@@ -39,6 +51,18 @@ def _make_renderer_mock(mm_limits: dict[str, int]) -> MagicMock:
51 renderer, BaseRenderer
52 )
53 renderer._clear_processor_cache = BaseRenderer._clear_processor_cache
54 + renderer.warmup_mm = BaseRenderer.warmup_mm.__get__(renderer, BaseRenderer)
55 + renderer.start_mm_warmup_in_background = (
56 + BaseRenderer.start_mm_warmup_in_background.__get__(renderer, BaseRenderer)
57 + )
58 + renderer._join_mm_warmup = BaseRenderer._join_mm_warmup.__get__(
59 + renderer, BaseRenderer
60 + )
61 + renderer.shutdown = BaseRenderer.shutdown.__get__(renderer, BaseRenderer)
62 + # No background warmup launched by default; warmup() takes the inline path.
63 + renderer._mm_warmup_future = None
64 + # MM warmup has not run yet; warmup_mm must actually execute on the mock.
65 + renderer._mm_warmup_done = False
66 renderer.clear_mm_cache = MagicMock()
67 renderer.model_config.max_model_len = 128
68 renderer.model_config.get_multimodal_config.return_value.limit_per_prompt = {}
@@ -88,7 +112,8 @@ class TestMmWarmupZeroLimitFiltering:
112
113
114 class TestMmWarmupRunsNormally:
91 - """MM warmup must run when mm_processor is set and limits > 0."""
115 + # MM warmup must run when mm_processor is set and limits > 0; the chat
116 + # template warmup must run alongside it.
117
118 def test_processor_apply_called(self):
119 renderer = _make_renderer_mock({"image": 1})
@@ -106,6 +131,14 @@ class TestMmWarmupRunsNormally:
131
132 renderer.clear_mm_cache.assert_called_once()
133
134 + def test_render_chat_called_with_warmup_message(self):
135 + renderer = _make_renderer_mock({"image": 1})
136 +
137 + with patch("vllm.multimodal.processing.TimingContext", autospec=True):
138 + BaseRenderer.warmup(renderer, ChatParams())
139 +
140 + renderer.render_chat.assert_called_once()
141 +
142
143 class TestMmWarmupSkippedWhenNoProcessor:
144 """MM warmup must be skipped when mm_processor is None (text-only model)."""
@@ -133,3 +166,315 @@ class TestReadonlyMmWarmup:
166
167 readonly_mm_processor.apply.assert_called_once()
168 readonly_mm_processor.cache.clear_cache.assert_called_once()
169 +
170 +
171 +class TestWarmupFaultIsolation:
172 + # A failure during a multimodal processor warmup is caught so it does not
173 + # abort the remaining warmup steps; warmup itself must not raise.
174 +
175 + def test_chat_failure_does_not_abort_mm_warmup(self):
176 + renderer = _make_renderer_mock({"image": 1})
177 + renderer.render_chat.side_effect = RuntimeError("chat boom")
178 +
179 + with patch("vllm.multimodal.processing.TimingContext", autospec=True):
180 + BaseRenderer.warmup(renderer, ChatParams()) # must not raise
181 +
182 + renderer.mm_processor.apply.assert_called_once()
183 +
184 + def test_mm_failure_does_not_abort_readonly_warmup(self):
185 + renderer = _make_renderer_mock({"image": 1})
186 + readonly_mm_processor = MagicMock()
187 + readonly_mm_processor.info.allowed_mm_limits = {"image": 1}
188 + renderer._readonly_mm_processor = readonly_mm_processor
189 + # main processor warmup blows up before apply()
190 + renderer.mm_processor.dummy_inputs.get_dummy_processor_inputs.side_effect = (
191 + RuntimeError("mm boom")
192 + )
193 +
194 + with patch("vllm.multimodal.processing.TimingContext", autospec=True):
195 + BaseRenderer.warmup(renderer, ChatParams()) # must not raise
196 +
197 + readonly_mm_processor.apply.assert_called_once()
198 + # cache is still cleared in the failed task's finally
199 + renderer.clear_mm_cache.assert_called_once()
200 +
201 +
202 +class TestBackgroundMmWarmup:
203 + # The multimodal warmup is launched as a task on the single-worker
204 + # _mm_executor so it overlaps engine-core init (fork) while staying
205 + # serialized with the serving path (_process_multimodal runs on the same
206 + # executor). warmup()/reset_mm_cache/shutdown join it. This also fixes the
207 + # numba workqueue "Concurrent access has been detected" abort: warmup and a
208 + # concurrent serving request can no longer both enter the numba parallel
209 + # region at once.
210 +
211 + def _make_bg_renderer(self, mm_limits: dict[str, int]):
212 + # A renderer mock with a real single-worker executor so submit()
213 + # returns a real Future and warmup_mm actually runs (in the worker).
214 + renderer = _make_renderer_mock(mm_limits)
215 + renderer._mm_executor = ThreadPoolExecutor(max_workers=1)
216 + return renderer
217 +
218 + def test_start_submits_future_to_mm_executor(self):
219 + renderer = self._make_bg_renderer({"image": 1})
220 + try:
221 + with patch("vllm.multimodal.processing.TimingContext", autospec=True):
222 + # Spy on the real executor's submit to confirm the warmup is
223 + # dispatched through _mm_executor (not a separate Thread).
224 + with patch.object(
225 + renderer._mm_executor, "submit", wraps=renderer._mm_executor.submit
226 + ) as spy:
227 + renderer.start_mm_warmup_in_background()
228 + assert isinstance(renderer._mm_warmup_future, Future)
229 + assert spy.called
230 + renderer._join_mm_warmup()
231 + assert renderer._mm_warmup_future is None
232 + renderer.mm_processor.apply.assert_called_once()
233 + finally:
234 + renderer._mm_executor.shutdown(wait=True)
235 +
236 + def test_start_noop_for_text_only_model(self):
237 + renderer = _make_renderer_mock({})
238 + renderer.mm_processor = None
239 + # _readonly_mm_processor is already None
240 +
241 + renderer.start_mm_warmup_in_background()
242 +
243 + assert renderer._mm_warmup_future is None
244 +
245 + def test_start_is_run_at_most_once(self):
246 + renderer = self._make_bg_renderer({"image": 1})
247 + try:
248 + with patch("vllm.multimodal.processing.TimingContext", autospec=True):
249 + renderer.start_mm_warmup_in_background()
250 + first = renderer._mm_warmup_future
251 + renderer.start_mm_warmup_in_background() # must not spawn a second
252 + assert renderer._mm_warmup_future is first
253 + renderer._join_mm_warmup()
254 + finally:
255 + renderer._mm_executor.shutdown(wait=True)
256 +
257 + def test_warmup_joins_background_and_does_not_rerun_mm(self):
258 + # When a background MM warmup is in flight, warmup() must join it and
259 + # run only the chat warmup — the MM warmup must not run twice.
260 + renderer = self._make_bg_renderer({"image": 1})
261 + try:
262 + with patch("vllm.multimodal.processing.TimingContext", autospec=True):
263 + renderer.start_mm_warmup_in_background()
264 + BaseRenderer.warmup(renderer, ChatParams())
265 +
266 + # MM apply called exactly once (by the background warmup task).
267 + renderer.mm_processor.apply.assert_called_once()
268 + # Chat warmup ran exactly once.
269 + renderer.render_chat.assert_called_once()
270 + # Background task has been joined and cleared.
271 + assert renderer._mm_warmup_future is None
272 + finally:
273 + renderer._mm_executor.shutdown(wait=True)
274 +
275 + def test_warmup_does_not_rerun_mm_after_reset_joins_background(self):
276 + # Regression: reset_mm_cache joins the background warmup before
277 + # warmup() runs (it clears _mm_warmup_future). warmup() must still not
278 + # re-run the MM warmup — the _mm_warmup_done flag survives the join.
279 + renderer = self._make_bg_renderer({"image": 1})
280 + try:
281 + with patch("vllm.multimodal.processing.TimingContext", autospec=True):
282 + renderer.start_mm_warmup_in_background()
283 + # Simulate reset_mm_cache joining the background warmup first
284 + # (clears _mm_warmup_future, sets _mm_warmup_done via the join).
285 + renderer._join_mm_warmup()
286 + assert renderer._mm_warmup_future is None # joined & cleared
287 + BaseRenderer.warmup(renderer, ChatParams())
288 +
289 + # MM apply called exactly once (by the background warmup task);
290 + # warmup() did not re-run warmup_mm despite the future being cleared.
291 + renderer.mm_processor.apply.assert_called_once()
292 + renderer.render_chat.assert_called_once()
293 + finally:
294 + renderer._mm_executor.shutdown(wait=True)
295 +
296 + def test_warmup_mm_runs_at_most_once(self):
297 + # Direct repeated calls to warmup_mm run the MM warmup only once.
298 + renderer = _make_renderer_mock({"image": 1})
299 +
300 + with patch("vllm.multimodal.processing.TimingContext", autospec=True):
301 + renderer.warmup_mm()
302 + renderer.warmup_mm() # second call must be a no-op
303 +
304 + renderer.mm_processor.apply.assert_called_once()
305 +
306 + def test_shutdown_joins_background_warmup(self):
307 + # shutdown() must join the background warmup before closing caches,
308 + # so the mm_processor_cache is never touched concurrently.
309 + renderer = self._make_bg_renderer({"image": 1})
310 +
311 + with patch("vllm.multimodal.processing.TimingContext", autospec=True):
312 + renderer.start_mm_warmup_in_background()
313 + future = renderer._mm_warmup_future
314 + renderer.shutdown()
315 +
316 + assert renderer._mm_warmup_future is None
317 + # The background warmup was allowed to complete (apply ran) and the
318 + # future is done after shutdown joined it.
319 + assert future.done()
320 + renderer.mm_processor.apply.assert_called_once()
321 +
322 +
323 +class TestEngineStartWarmupHook:
324 + # The renderer is handed to the client, which starts the MM warmup
325 + # (renderer.start_mm_warmup_in_background) only after engine-core
326 + # processes have been forked — a live warmup thread at fork() time would
327 + # deadlock the child (it inherits locks whose owning thread vanishes).
328 + # These tests pin the renderer plumbing contract on EngineCoreClient.
329 +
330 + def _mock_config(self):
331 + from types import SimpleNamespace
332 +
333 + from vllm.v1.engine.core_client import EngineCoreClient # noqa: F401
334 +
335 + cfg = SimpleNamespace(
336 + parallel_config=SimpleNamespace(
337 + data_parallel_size=1,
338 + data_parallel_external_lb=False,
339 + ),
340 + model_config=SimpleNamespace(multimodal_config=None),
341 + )
342 + return cfg
343 +
344 + def _mock_renderer(self):
345 + return MagicMock()
346 +
347 + def test_inproc_client_rejects_renderer(self):
348 + # InprocClient has no _start_mm_warmup (only MPClient does), so it
349 + # takes no renderer at all: passing one is a TypeError at the call
350 + # site rather than a silently-ignored kwarg.
351 + from vllm.v1.engine import core_client as cc
352 +
353 + with pytest.raises(TypeError, match="renderer"):
354 + cc.InprocClient(
355 + MagicMock(),
356 + MagicMock(),
357 + MagicMock(),
358 + renderer=self._mock_renderer(),
359 + )
360 +
361 + def test_inproc_client_forwards_fail_callback_but_not_renderer(self):
362 + # EngineCore args (incl. executor_fail_callback) pass through; the
363 + # renderer kwarg is intercepted by InprocClient.__init__ (keyword-only
364 + # after *), so it is never forwarded to EngineCore — which has no
365 + # renderer parameter at all.
366 + from vllm.v1.engine import core_client as cc
367 +
368 + callback = MagicMock()
369 + with patch.object(cc, "EngineCore") as mock_engine_core:
370 + cc.InprocClient(
371 + MagicMock(),
372 + MagicMock(),
373 + MagicMock(),
374 + executor_fail_callback=callback,
375 + )
376 +
377 + mock_engine_core.assert_called_once()
378 + args, kwargs = mock_engine_core.call_args
379 + assert len(args) == 3 # vllm_config, executor_class, log_stats
380 + assert "renderer" not in kwargs
381 + assert kwargs.get("executor_fail_callback") is callback
382 +
383 + def test_make_client_does_not_pass_renderer_to_inproc_client(self):
384 + # In-process: the renderer must never reach InprocClient, so MM
385 + # warmup stays a plain inline call inside renderer.warmup().
386 + from vllm.v1.engine import core_client as cc
387 +
388 + cfg = self._mock_config()
389 + renderer = self._mock_renderer()
390 +
391 + with (
392 + patch.object(cc, "InprocClient") as mock_inproc,
393 + patch.object(cc, "SyncMPClient") as mock_sync,
394 + ):
395 + client = cc.EngineCoreClient.make_client(
396 + multiprocess_mode=False,
397 + asyncio_mode=False,
398 + vllm_config=cfg,
399 + executor_class=MagicMock(),
400 + log_stats=False,
401 + renderer=renderer,
402 + )
403 +
404 + assert client is mock_inproc.return_value
405 + mock_inproc.assert_called_once()
406 + assert "renderer" not in mock_inproc.call_args.kwargs
407 + mock_sync.assert_not_called()
408 + renderer.start_mm_warmup_in_background.assert_not_called()
409 +
410 + @pytest.mark.parametrize(
411 + ("client_cls", "multiprocess_mode"),
412 + [
413 + # SyncMPClient forks engine-core: the renderer must be handed to
414 + # it so the warmup fires after proc.start(), not before fork().
415 + ("SyncMPClient", True),
416 + ],
417 + )
418 + def test_make_client_passes_renderer(self, client_cls, multiprocess_mode):
419 + from vllm.v1.engine import core_client as cc
420 +
421 + cfg = self._mock_config()
422 + renderer = self._mock_renderer()
423 +
424 + with patch.object(cc, client_cls) as mock_client:
425 + cc.EngineCoreClient.make_client(
426 + multiprocess_mode=multiprocess_mode,
427 + asyncio_mode=False,
428 + vllm_config=cfg,
429 + executor_class=MagicMock(),
430 + log_stats=False,
431 + renderer=renderer,
432 + )
433 +
434 + mock_client.assert_called_once()
435 + assert mock_client.call_args.kwargs["renderer"] is renderer
436 +
437 + @pytest.mark.parametrize(
438 + ("client_cls", "dp_size", "external_lb"),
439 + [
440 + ("AsyncMPClient", 1, False),
441 + ("DPAsyncMPClient", 2, True),
442 + ],
443 + )
444 + def test_make_async_mp_client_passes_renderer(
445 + self, client_cls, dp_size, external_lb
446 + ):
447 + from vllm.v1.engine import core_client as cc
448 +
449 + cfg = self._mock_config()
450 + cfg.parallel_config.data_parallel_size = dp_size
451 + cfg.parallel_config.data_parallel_external_lb = external_lb
452 + renderer = self._mock_renderer()
453 +
454 + with patch.object(cc, client_cls) as mock_client:
455 + cc.EngineCoreClient.make_async_mp_client(
456 + cfg,
457 + executor_class=MagicMock(),
458 + log_stats=False,
459 + renderer=renderer,
460 + )
461 +
462 + mock_client.assert_called_once()
463 + assert mock_client.call_args.kwargs["renderer"] is renderer
464 +
465 + def test_mp_client_starts_mm_warmup_only_with_renderer(self):
466 + # MPClient must call renderer.start_mm_warmup_in_background when a
467 + # renderer was given, and skip it (no crash) when not.
468 + from vllm.v1.engine import core_client as cc
469 +
470 + # No renderer: _start_mm_warmup is a no-op.
471 + client = cc.MPClient.__new__(cc.MPClient)
472 + client._renderer = None
473 + client._start_mm_warmup()
474 +
475 + # With a renderer: it starts the background MM warmup.
476 + renderer = self._mock_renderer()
477 + client2 = cc.MPClient.__new__(cc.MPClient)
478 + client2._renderer = renderer
479 + client2._start_mm_warmup()
480 + renderer.start_mm_warmup_in_background.assert_called_once()
vllm/renderers/base.py
+117 -24
@@ -5,7 +5,7 @@ import time
5 import weakref
6 from abc import ABC, abstractmethod
7 from collections.abc import Mapping, Sequence
8 -from concurrent.futures import ThreadPoolExecutor
8 +from concurrent.futures import Future, ThreadPoolExecutor
9 from contextlib import ExitStack
10 from functools import cached_property
11 from typing import TYPE_CHECKING, Any, Generic, overload
@@ -85,6 +85,26 @@ class BaseRenderer(ABC, Generic[_T]):
85
86 self.tokenizer = tokenizer
87
88 + # Background multimodal warmup task (a future returned by
89 + # _mm_executor) so it overlaps engine-core initialization while
90 + # staying serialized with the serving path through the same
91 + # single-worker executor. None when no background warmup has been
92 + # launched (or it has already been joined). Routing the warmup through
93 + # _mm_executor (max_workers=1) guarantees it can never run
94 + # concurrently with a serving request inside the numba workqueue
95 + # parallel region, which would otherwise hit numba's
96 + # "Concurrent access has been detected" fatal abort (see
97 + # vllm/utils/jit_monitor.py, which forces the workqueue layer).
98 + # Joined by the first of reset_mm_cache / warmup / shutdown so the
99 + # mm_processor_cache is also never touched concurrently by the warmup
100 + # and the serving path.
101 + self._mm_warmup_future: Future[None] | None = None
102 + # Whether warmup_mm has already run. Survives _join_mm_warmup (which
103 + # clears the thread pointer), so warmup() can tell that the background
104 + # warmup already ran even after reset_mm_cache joined it first, and
105 + # avoid running warmup_mm a second time.
106 + self._mm_warmup_done: bool = False
107 +
108 # Thread pool executor for blocking tokenizer operations. The
109 # multimodal processor receives a deep-copied tokenizer (see #36557)
110 # so it is safe to run tokenization and MM preprocessing concurrently.
@@ -243,12 +263,97 @@ class BaseRenderer(ABC, Generic[_T]):
263 elapsed = time.perf_counter() - start_time
264 logger.info("%s warmup completed in %.3fs", log_prefix, elapsed)
265
266 + def warmup_mm(self) -> None:
267 + # Run only the multimodal warmup (mm + readonly mm), serially.
268 + # Idempotent: runs at most once. A background warmup
269 + # (start_mm_warmup_in_background) may already have run it, and
270 + # reset_mm_cache may have joined that background warmup task before
271 + # warmup() reaches this point — the _mm_warmup_done flag survives the
272 + # join (the future pointer does not), so warmup() can still tell it
273 + # already ran.
274 + if self._mm_warmup_done:
275 + return
276 + try:
277 + # prevent MM processor hangs
278 + with set_default_torch_num_threads(1):
279 + if self.mm_processor:
280 + try:
281 + logger.debug("Warming up multi-modal processing...")
282 + self._warmup_mm_processor(
283 + self.mm_processor,
284 + log_prefix="Multi-modal",
285 + )
286 + except Exception:
287 + logger.warning("Multi-modal warmup failed")
288 + finally:
289 + self.clear_mm_cache()
290 +
291 + if self._readonly_mm_processor is not None:
292 + try:
293 + logger.debug("Warming up readonly multi-modal processing...")
294 + self._warmup_mm_processor(
295 + self._readonly_mm_processor,
296 + log_prefix="Readonly multi-modal",
297 + )
298 + except Exception:
299 + logger.warning("Readonly multi-modal warmup failed")
300 + finally:
301 + self._clear_processor_cache(self._readonly_mm_processor)
302 + finally:
303 + self._mm_warmup_done = True
304 +
305 + def start_mm_warmup_in_background(self) -> None:
306 + # Launch the multimodal warmup as a task on the single-worker
307 + # _mm_executor so it overlaps with engine-core initialization
308 + # (model loading) while staying serialized with the serving path.
309 + #
310 + # Why the same executor as serving (not a separate Thread):
311 + # _process_multimodal runs on _mm_executor (max_workers=1) and
312 + # enters the numba workqueue parallel region (e.g. Kimi K2.5 vision
313 + # preprocessing via kimi_k25_vision_fused). A separate warmup Thread
314 + # would run concurrently with that single serving worker, so both
315 + # threads would enter the numba workqueue parallel region at the same
316 + # time and trip numba's "Concurrent access has been detected" ->
317 + # "Fatal Python error: Aborted" (see vllm/utils/jit_monitor.py, which
318 + # forces the non-threadsafe workqueue layer). Submitting the warmup to
319 + # the same single-worker executor makes warmup and serving
320 + # physically incapable of overlapping on the numba threading layer,
321 + # while still overlapping at the process level with the (forked)
322 + # engine-core init in the main thread.
323 + #
324 + # The warmup task is joined by the first of: reset_mm_cache, warmup, or
325 + # shutdown. This also guarantees the frontend mm_processor_cache —
326 + # which is not designed for concurrent access (its get_and_update must
327 + # stay ordered with the engine-core cache) — is never touched
328 + # concurrently by the warmup and the serving path.
329 + if self.mm_processor is None and self._readonly_mm_processor is None:
330 + return # text-only model: nothing to warm up
331 + if self._mm_warmup_future is not None:
332 + return # already launched
333 + self._mm_warmup_future = self._mm_executor.submit(self.warmup_mm)
334 +
335 + def _join_mm_warmup(self) -> None:
336 + # Wait for the background MM warmup task to finish, if one was started.
337 + # Called from the main / event-loop thread (reset_mm_cache, warmup,
338 + # shutdown) — never from inside the _mm_executor worker — so
339 + # waiting on the future cannot deadlock.
340 + future = getattr(self, "_mm_warmup_future", None)
341 + if future is not None:
342 + future.result()
343 + self._mm_warmup_future = None
344 +
345 def warmup(self, chat_params: ChatParams) -> None:
346 """
347 Warm up this renderer to avoid first-request latency.
348
349 For chat requests:
350 - Jinja2 template compilation
351 +
352 + If a background multimodal warmup was launched (to overlap
353 + engine-core init) it is joined first so it is not still touching the
354 + mm_processor_cache. warmup_mm is idempotent, so the call below is a
355 + no-op if the background warmup — or a prior reset_mm_cache that joined
356 + it — already ran it.
357 """
358 from vllm.entrypoints.chat_utils import ChatTemplateResolutionError
359
@@ -267,29 +372,12 @@ class BaseRenderer(ABC, Generic[_T]):
372 except Exception:
373 logger.warning("Chat template warmup failed", exc_info=True)
374
270 - if self.mm_processor:
271 - try:
272 - logger.debug("Warming up multi-modal processing...")
273 - self._warmup_mm_processor(
274 - self.mm_processor,
275 - log_prefix="Multi-modal",
276 - )
277 - except Exception:
278 - logger.warning("Multi-modal warmup failed")
279 - finally:
280 - self.clear_mm_cache()
281 -
282 - if self._readonly_mm_processor is not None:
283 - try:
284 - logger.debug("Warming up readonly multi-modal processing...")
285 - self._warmup_mm_processor(
286 - self._readonly_mm_processor,
287 - log_prefix="Readonly multi-modal",
288 - )
289 - except Exception:
290 - logger.warning("Readonly multi-modal warmup failed")
291 - finally:
292 - self._clear_processor_cache(self._readonly_mm_processor)
375 + if self._mm_warmup_future is not None:
376 + # Join a background MM warmup launched to overlap engine-core init.
377 + self._join_mm_warmup()
378 + # Idempotent: no-op if the background warmup (or a prior reset_mm_cache)
379 + # already ran the MM warmup.
380 + self.warmup_mm()
381
382 async def clear_mm_cache_async(self) -> None:
383 """Serialize clear_mm_cache through the multimodal executor to avoid
@@ -297,6 +385,11 @@ class BaseRenderer(ABC, Generic[_T]):
385 await self._clear_mm_cache_async()
386
387 def shutdown(self) -> None:
388 + # Wait for any background MM warmup to finish first: it runs on the
389 + # _mm_executor and touches the mm_processor_cache, so it must be
390 + # quiescent before _resources.close() tears down the executors and
391 + # the cache (both are registered as resources in __init__).
392 + self._join_mm_warmup()
393 self._resources.close()
394
395 def get_bos_token_id(self) -> int | None:
vllm/v1/engine/async_llm.py
+8
@@ -153,6 +153,10 @@ class AsyncLLM(EngineClient):
153 )
154
155 # EngineCore (starts the engine in background process).
156 + # Hand the renderer to the client so it can start the frontend MM
157 + # warmup only after engine-core fork (the why is in
158 + # BaseRenderer.start_mm_warmup_in_background). The warmup is joined
159 + # by reset_mm_cache / warmup / shutdown.
160 self.engine_core = EngineCoreClient.make_async_mp_client(
161 vllm_config=vllm_config,
162 executor_class=executor_class,
@@ -160,6 +164,7 @@ class AsyncLLM(EngineClient):
164 client_addresses=client_addresses,
165 client_count=client_count,
166 client_index=client_index,
167 + renderer=renderer,
168 )
169
170 # Loggers.
@@ -1023,6 +1028,9 @@ class AsyncLLM(EngineClient):
1028 await asyncio.gather(*coros)
1029
1030 async def reset_mm_cache(self) -> None:
1031 + # Join the background MM warmup first: the mm_processor_cache is not
1032 + # safe for concurrent access with its apply/clear.
1033 + self.renderer._join_mm_warmup()
1034 await self.renderer.clear_mm_cache_async()
1035 await self.engine_core.reset_mm_cache_async()
1036
vllm/v1/engine/core_client.py
+82 -8
@@ -26,6 +26,7 @@ from vllm.config import VllmConfig
26 from vllm.envs import VLLM_ENGINE_READY_TIMEOUT_S
27 from vllm.logger import init_logger
28 from vllm.lora.request import LoRARequest
29 +from vllm.renderers import BaseRenderer
30 from vllm.tasks import SupportedTask
31 from vllm.tracing import instrument
32 from vllm.utils.async_utils import in_loop
@@ -93,7 +94,15 @@ class EngineCoreClient(ABC):
94 vllm_config: VllmConfig,
95 executor_class: type[Executor],
96 log_stats: bool,
97 + renderer: BaseRenderer | None = None,
98 ) -> "EngineCoreClient":
99 + # renderer is passed through to the multiprocess clients, which start
100 + # the frontend MM warmup (renderer.start_mm_warmup_in_background) once
101 + # the engine-core processes have been forked, so warmup overlaps
102 + # engine-core load without a live thread at fork() time. In-process
103 + # clients do not take a renderer: the engine core is built in this
104 + # process, so there is no other process to overlap with and the MM
105 + # warmup runs synchronously inside renderer.warmup() instead.
106 # TODO: support this for debugging purposes.
107 if asyncio_mode and not multiprocess_mode:
108 raise NotImplementedError(
@@ -103,11 +112,19 @@ class EngineCoreClient(ABC):
112
113 if multiprocess_mode and asyncio_mode:
114 return EngineCoreClient.make_async_mp_client(
106 - vllm_config, executor_class, log_stats
115 + vllm_config,
116 + executor_class,
117 + log_stats,
118 + renderer=renderer,
119 )
120
121 if multiprocess_mode and not asyncio_mode:
110 - return SyncMPClient(vllm_config, executor_class, log_stats)
122 + return SyncMPClient(
123 + vllm_config,
124 + executor_class,
125 + log_stats,
126 + renderer=renderer,
127 + )
128
129 return InprocClient(vllm_config, executor_class, log_stats)
130
@@ -120,6 +137,7 @@ class EngineCoreClient(ABC):
137 client_addresses: dict[str, Any] | None = None,
138 client_count: int = 1,
139 client_index: int = 0,
140 + renderer: BaseRenderer | None = None,
141 ) -> "AsyncMPClient":
142 parallel_config = vllm_config.parallel_config
143 client_args = (
@@ -133,10 +151,19 @@ class EngineCoreClient(ABC):
151 if parallel_config.data_parallel_size > 1:
152 if parallel_config.data_parallel_external_lb:
153 # External load balancer - client per DP rank.
136 - return DPAsyncMPClient(*client_args)
154 + return DPAsyncMPClient(
155 + *client_args,
156 + renderer=renderer,
157 + )
158 # Internal load balancer - client balances to all DP ranks.
138 - return DPLBAsyncMPClient(*client_args)
139 - return AsyncMPClient(*client_args)
159 + return DPLBAsyncMPClient(
160 + *client_args,
161 + renderer=renderer,
162 + )
163 + return AsyncMPClient(
164 + *client_args,
165 + renderer=renderer,
166 + )
167
168 @abstractmethod
169 def shutdown(self, timeout: float | None = None) -> None: ...
@@ -313,8 +340,23 @@ class InprocClient(EngineCoreClient):
340 * pulls EngineCoreOutputs by stepping the EngineCore
341 """
342
316 - def __init__(self, *args, **kwargs):
317 - self.engine_core = EngineCore(*args, **kwargs)
343 + # Takes no renderer: this class has no _start_mm_warmup (only MPClient
344 + # does), so a renderer would never be used here. EngineCore is built in
345 + # this process and MM warmup runs synchronously inside renderer.warmup().
346 + def __init__(
347 + self,
348 + vllm_config: VllmConfig,
349 + executor_class: type[Executor],
350 + log_stats: bool,
351 + *,
352 + executor_fail_callback: Callable | None = None,
353 + ):
354 + self.engine_core = EngineCore(
355 + vllm_config,
356 + executor_class,
357 + log_stats,
358 + executor_fail_callback=executor_fail_callback,
359 + )
360
361 def get_output(self) -> EngineCoreOutputs:
362 outputs, model_executed = self.engine_core.step_fn()
@@ -520,8 +562,10 @@ class MPClient(EngineCoreClient):
562 executor_class: type[Executor],
563 log_stats: bool,
564 client_addresses: dict[str, Any] | None = None,
565 + renderer: BaseRenderer | None = None,
566 ):
567 self.vllm_config = vllm_config
568 + self._renderer: BaseRenderer | None = renderer
569
570 # ZMQ setup.
571 sync_ctx = zmq.Context(io_threads=2)
@@ -583,6 +627,9 @@ class MPClient(EngineCoreClient):
627 )
628 finally:
629 actual_address_pipe.close()
630 + # Engines are managed externally: this process does not fork
631 + # them, so there is no fork race; start the MM warmup now.
632 + self._start_mm_warmup()
633 else:
634 # Engines are managed by this client.
635 addresses = get_engine_zmq_addresses(vllm_config)
@@ -614,6 +661,14 @@ class MPClient(EngineCoreClient):
661 coordinator = engine_launch.coordinator
662 addresses = engine_launch.addresses
663 tensor_queue = engine_launch.tensor_queue
664 + # Engine-core processes have now all been forked/started
665 + # (CoreEngineProcManager.proc.start()). It is now safe to
666 + # launch the frontend background MM warmup: it must not
667 + # run while fork() is in flight (a live thread holding a
668 + # lock would deadlock the forked child), but the
669 + # engine-core model load (minutes) that follows is exactly
670 + # what the warmup should overlap with.
671 + self._start_mm_warmup()
672
673 self.stats_update_address = addresses.frontend_stats_publish_address
674 if coordinator is not None:
@@ -708,6 +763,14 @@ class MPClient(EngineCoreClient):
763 def dp_engines_running(self) -> bool:
764 return self.engines_running
765
766 + def _start_mm_warmup(self) -> None:
767 + # Called once the engine-core process(es) have been created (forked or
768 + # externally managed). Overlap the frontend MM warmup with the
769 + # engine-core model load. This is a no-op when no renderer was passed
770 + # (e.g. text-only serving or tests).
771 + if self._renderer is not None:
772 + self._renderer.start_mm_warmup_in_background()
773 +
774 def start_engine_core_monitor(self):
775 """Start a monitor thread for engine core processes."""
776 engine_manager = self.resources.engine_manager
@@ -808,13 +871,18 @@ class SyncMPClient(MPClient):
871
872 @instrument(span_name="SyncMPClient init")
873 def __init__(
811 - self, vllm_config: VllmConfig, executor_class: type[Executor], log_stats: bool
874 + self,
875 + vllm_config: VllmConfig,
876 + executor_class: type[Executor],
877 + log_stats: bool,
878 + renderer: BaseRenderer | None = None,
879 ):
880 super().__init__(
881 asyncio_mode=False,
882 vllm_config=vllm_config,
883 executor_class=executor_class,
884 log_stats=log_stats,
885 + renderer=renderer,
886 )
887
888 self.is_dp = self.vllm_config.parallel_config.data_parallel_size > 1
@@ -987,6 +1055,7 @@ class AsyncMPClient(MPClient):
1055 client_addresses: dict[str, Any] | None = None,
1056 client_count: int = 1,
1057 client_index: int = 0,
1058 + renderer: BaseRenderer | None = None,
1059 ):
1060 super().__init__(
1061 asyncio_mode=True,
@@ -994,6 +1063,7 @@ class AsyncMPClient(MPClient):
1063 executor_class=executor_class,
1064 log_stats=log_stats,
1065 client_addresses=client_addresses,
1066 + renderer=renderer,
1067 )
1068
1069 self.client_count = client_count
@@ -1262,6 +1332,7 @@ class DPAsyncMPClient(AsyncMPClient):
1332 client_addresses: dict[str, Any] | None = None,
1333 client_count: int = 1,
1334 client_index: int = 0,
1335 + renderer: BaseRenderer | None = None,
1336 ):
1337 self.current_wave = 0
1338
@@ -1272,6 +1343,7 @@ class DPAsyncMPClient(AsyncMPClient):
1343 client_addresses,
1344 client_count,
1345 client_index,
1346 + renderer,
1347 )
1348
1349 # List of [waiting, running, kv_cache_usage] per engine.
@@ -1444,6 +1516,7 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
1516 client_addresses: dict[str, Any] | None = None,
1517 client_count: int = 1,
1518 client_index: int = 0,
1519 + renderer: BaseRenderer | None = None,
1520 ):
1521 self.client_count = client_count
1522
@@ -1460,6 +1533,7 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
1533 client_addresses,
1534 client_count,
1535 client_index,
1536 + renderer,
1537 )
1538
1539 assert len(self.core_engines) > 1
vllm/v1/engine/llm_engine.py
+8
@@ -102,12 +102,17 @@ class LLMEngine:
102 )
103
104 # EngineCore (gets EngineCoreRequests and gives EngineCoreOutputs)
105 + # Hand the renderer to the client. In multiprocess mode the client
106 + # starts the MM warmup only after engine-core fork (the why is in
107 + # BaseRenderer.start_mm_warmup_in_background); InprocClient takes no
108 + # renderer, so MM warmup stays inside renderer.warmup() there.
109 self.engine_core = EngineCoreClient.make_client(
110 multiprocess_mode=multiprocess_mode,
111 asyncio_mode=False,
112 vllm_config=vllm_config,
113 executor_class=executor_class,
114 log_stats=self.log_stats,
115 + renderer=renderer,
116 )
117
118 self.logger_manager: StatLoggerManager | None = None
@@ -343,6 +348,9 @@ class LLMEngine:
348 self.engine_core.profile(False)
349
350 def reset_mm_cache(self):
351 + # Join the background MM warmup first: the mm_processor_cache is not
352 + # safe for concurrent access with its apply/clear.
353 + self.renderer._join_mm_warmup()
354 self.renderer.clear_mm_cache()
355 self.engine_core.reset_mm_cache()
356