[Perf] Isolate MM preprocessing on its own executor (#49524)
Signed-off-by: Guan-Ming (Wesley) Chiu <105915352+guan404ming@users.noreply.github.com>
Guan-Ming Chiu committed
Jul 26, 2026 at 16:04 UTC
8d28b48d01b2ba56e962c7c57b894c6b4fcf8a35
3 files changed
+31
-20
tests/test_config.py
+19
-8
@@ -1558,20 +1558,31 @@ def test_fault_tolerance_requires_single_api_server():
1558
1559
1560
def test_renderer_num_workers_with_mm_cache():
1561
- """Disallow renderer_num_workers > 1 when mm processor cache is enabled,
1562
- since neither cache type is thread-safe."""
1561
+ """Disallow renderer_num_workers > 1 with the mm processor cache only for
1562
+ pooling models, whose preprocessing runs on the renderer workers."""
1563
mm_model = "Qwen/Qwen2-VL-2B-Instruct"
1564
1565
- # Should raise: multi-worker + cache enabled (default cache_gb=4)
1565
+ # Should raise: pooling + multi-worker + cache enabled (default cache_gb=4)
1566
with pytest.raises(ValueError, match="renderer-num-workers"):
1567
- ModelConfig(mm_model, renderer_num_workers=4)
1567
+ ModelConfig(mm_model, runner="pooling", renderer_num_workers=4)
1568
1569
- # Should raise: multi-worker + explicit cache size
1569
+ # Should raise: pooling + multi-worker + explicit cache size
1570
with pytest.raises(ValueError, match="renderer-num-workers"):
1571
- ModelConfig(mm_model, renderer_num_workers=2, mm_processor_cache_gb=1.0)
1571
+ ModelConfig(
1572
+ mm_model,
1573
+ runner="pooling",
1574
+ renderer_num_workers=2,
1575
+ mm_processor_cache_gb=1.0,
1576
+ )
1577
+
1578
+ # Should pass: pooling + multi-worker + cache disabled
1579
+ config = ModelConfig(
1580
+ mm_model, runner="pooling", renderer_num_workers=4, mm_processor_cache_gb=0
1581
+ )
1582
+ assert config.renderer_num_workers == 4
1583
1573
- # Should pass: multi-worker + cache disabled
1574
- config = ModelConfig(mm_model, renderer_num_workers=4, mm_processor_cache_gb=0)
1584
+ # Should pass: generate models preprocess on the dedicated mm executor
1585
+ config = ModelConfig(mm_model, renderer_num_workers=4)
1586
assert config.renderer_num_workers == 4
1587
1588
# Should pass: single worker + cache enabled (default)
vllm/config/model.py
+4
-3
@@ -748,12 +748,13 @@ class ModelConfig:
748
if (
749
self.renderer_num_workers > 1
750
and self.multimodal_config.mm_processor_cache_gb > 0
751
+ and self.runner_type == "pooling"
752
):
753
raise ValueError(
754
"Cannot use --renderer-num-workers > 1 with the "
754
- "multimodal processor cache enabled. The cache is "
755
- "not thread-safe and does not support concurrent "
756
- "renderer workers. Please set "
755
+ "multimodal processor cache enabled for pooling models. "
756
+ "Pooling preprocessing runs on the renderer workers, and "
757
+ "the cache is not thread-safe. Please set "
758
"--renderer-num-workers 1 (the default), or "
759
"disable the cache with --mm-processor-cache-gb 0."
760
)
vllm/renderers/base.py
+8
-9
@@ -79,16 +79,15 @@ class BaseRenderer(ABC, Generic[_T]):
79
80
self.tokenizer = tokenizer
81
82
- # Shared thread pool executor for blocking tokenizer and
83
- # multimodal preprocessing operations. The multimodal processor
84
- # receives a deep-copied tokenizer (see #36557) so it is safe to
85
- # run tokenization and MM preprocessing concurrently.
82
+ # Thread pool executor for blocking tokenizer operations. The
83
+ # multimodal processor receives a deep-copied tokenizer (see #36557)
84
+ # so it is safe to run tokenization and MM preprocessing concurrently.
85
pool_workers = config.model_config.renderer_num_workers
86
self._executor = ThreadPoolExecutor(max_workers=pool_workers)
87
89
- # Multimodal preprocessing is always offloaded to the thread pool
90
- # to keep the asyncio event loop responsive under concurrent load.
91
- self._mm_executor: Executor = self._executor
88
+ # Separate single-worker executor so tokenization never queues behind
89
+ # MM preprocessing; must stay single-worker per #38418 (P0/P1 order).
90
+ self._mm_executor: Executor = ThreadPoolExecutor(max_workers=1)
91
92
# Offload tokenization to the thread pool. The sync
93
# ``_tokenize_prompt`` already encapsulates the unified ``__call__``
@@ -103,7 +102,7 @@ class BaseRenderer(ABC, Generic[_T]):
102
self._readonly_mm_processor: BaseMultiModalProcessor | None = None
103
self._mm_cache_stats: MultiModalCacheStats | None = None
104
self._clear_mm_cache_async = make_async(
106
- self.clear_mm_cache, executor=self._executor
105
+ self.clear_mm_cache, executor=self._mm_executor
106
)
107
self._process_multimodal_async = make_async(
108
self._process_multimodal, executor=self._mm_executor
@@ -282,7 +281,7 @@ class BaseRenderer(ABC, Generic[_T]):
281
self._clear_processor_cache(self._readonly_mm_processor)
282
283
async def clear_mm_cache_async(self) -> None:
285
- """Serialize clear_mm_cache through the shared executor to avoid
284
+ """Serialize clear_mm_cache through the multimodal executor to avoid
285
races with concurrent process_inputs on the mm_processor_cache."""
286
await self._clear_mm_cache_async()
287