A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 15:00 UTC
[Feature] Warm up readonly multimodal processor during renderer startup (#40797)
Signed-off-by: Chenguang ZHENG <645327136@qq.com> Co-authored-by: OpenAI Codex <codex@openai.com>
Chenguang Zheng committed
Apr 25, 2026 at 11:57 UTC
07351e0883470724dd5a7e9730ed10e01fc99d08
7 files changed
+368
-27
tests/entrypoints/openai/chat_completion/test_chat_error.py
+52
@@ -164,6 +164,58 @@ async def test_chat_error_non_stream():
164
await serving_chat.create_chat_completion(request)
165
166
167
+@pytest.mark.asyncio
168
+async def test_openai_chat_keeps_mm_cache_for_engine_execution():
169
+ mock_engine = MagicMock(spec=AsyncLLM)
170
+ mock_engine.errored = False
171
+ mock_engine.model_config = MockModelConfig()
172
+ mock_engine.input_processor = MagicMock()
173
+ mock_engine.renderer = _build_renderer(mock_engine.model_config)
174
+
175
+ serving_chat = _build_serving_chat(mock_engine)
176
+
177
+ request = ChatCompletionRequest(
178
+ model=MODEL_NAME,
179
+ messages=[{"role": "user", "content": "Test prompt"}],
180
+ )
181
+
182
+ result = await serving_chat.render_chat_request(request)
183
+
184
+ assert isinstance(result, tuple)
185
+ assert (
186
+ serving_chat.openai_serving_render.preprocess_chat.call_args.kwargs[
187
+ "skip_mm_cache"
188
+ ]
189
+ is False
190
+ )
191
+
192
+
193
+@pytest.mark.asyncio
194
+async def test_renderer_only_chat_request_skips_mm_cache():
195
+ mock_engine = MagicMock(spec=AsyncLLM)
196
+ mock_engine.errored = False
197
+ mock_engine.model_config = MockModelConfig()
198
+ mock_engine.input_processor = MagicMock()
199
+ mock_engine.renderer = _build_renderer(mock_engine.model_config)
200
+
201
+ serving_chat = _build_serving_chat(mock_engine)
202
+
203
+ request = ChatCompletionRequest(
204
+ model=MODEL_NAME,
205
+ messages=[{"role": "user", "content": "Test prompt"}],
206
+ )
207
+
208
+ result = await serving_chat.openai_serving_render.render_chat_request(request)
209
+
210
+ assert result.token_ids == [1, 2, 3]
211
+ assert (
212
+ serving_chat.openai_serving_render.preprocess_chat.call_args.kwargs[
213
+ "skip_mm_cache"
214
+ ]
215
+ is True
216
+ )
217
+
218
+
219
@pytest.mark.asyncio
220
async def test_chat_error_stream():
221
"""test finish_reason='error' returns 500 InternalServerError (streaming)"""
tests/entrypoints/openai/completion/test_completion_error.py
+61
-1
@@ -3,7 +3,7 @@
3
4
from dataclasses import dataclass, field
5
from typing import Any
6
-from unittest.mock import MagicMock
6
+from unittest.mock import AsyncMock, MagicMock
7
8
import pytest
9
@@ -148,6 +148,66 @@ async def test_completion_error_non_stream():
148
await serving_completion.create_completion(request)
149
150
151
+@pytest.mark.asyncio
152
+async def test_openai_completion_keeps_mm_cache_for_engine_execution():
153
+ mock_engine = MagicMock(spec=AsyncLLM)
154
+ mock_engine.errored = False
155
+ mock_engine.model_config = MockModelConfig()
156
+ mock_engine.input_processor = MagicMock()
157
+ mock_engine.renderer = _build_renderer(mock_engine.model_config)
158
+
159
+ serving_completion = _build_serving_completion(mock_engine)
160
+ serving_completion.openai_serving_render.preprocess_completion = AsyncMock(
161
+ return_value=[{"prompt_token_ids": [1, 2, 3]}]
162
+ )
163
+
164
+ request = CompletionRequest(
165
+ model=MODEL_NAME,
166
+ prompt="Test prompt",
167
+ )
168
+
169
+ result = await serving_completion.render_completion_request(request)
170
+
171
+ assert isinstance(result, list)
172
+ assert (
173
+ serving_completion.openai_serving_render.preprocess_completion.call_args.kwargs[
174
+ "skip_mm_cache"
175
+ ]
176
+ is False
177
+ )
178
+
179
+
180
+@pytest.mark.asyncio
181
+async def test_renderer_only_completion_request_skips_mm_cache():
182
+ mock_engine = MagicMock(spec=AsyncLLM)
183
+ mock_engine.errored = False
184
+ mock_engine.model_config = MockModelConfig()
185
+ mock_engine.input_processor = MagicMock()
186
+ mock_engine.renderer = _build_renderer(mock_engine.model_config)
187
+
188
+ serving_completion = _build_serving_completion(mock_engine)
189
+ serving_completion.openai_serving_render.preprocess_completion = AsyncMock(
190
+ return_value=[{"prompt_token_ids": [1, 2, 3]}]
191
+ )
192
+
193
+ request = CompletionRequest(
194
+ model=MODEL_NAME,
195
+ prompt="Test prompt",
196
+ )
197
+
198
+ result = await serving_completion.openai_serving_render.render_completion_request(
199
+ request
200
+ )
201
+
202
+ assert isinstance(result, list)
203
+ assert (
204
+ serving_completion.openai_serving_render.preprocess_completion.call_args.kwargs[
205
+ "skip_mm_cache"
206
+ ]
207
+ is True
208
+ )
209
+
210
+
211
@pytest.mark.asyncio
212
async def test_completion_error_stream():
213
"""test finish_reason='error' returns 500 InternalServerError (streaming)"""
tests/entrypoints/serve/disagg/test_generate_stream.py
+34
-1
@@ -12,7 +12,10 @@ from vllm.config.multimodal import MultiModalConfig
12
from vllm.entrypoints.openai.engine.protocol import StreamOptions
13
from vllm.entrypoints.openai.models.protocol import BaseModelPath
14
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
15
-from vllm.entrypoints.serve.disagg.protocol import GenerateRequest
15
+from vllm.entrypoints.serve.disagg.protocol import (
16
+ GenerateRequest,
17
+ GenerateResponse,
18
+)
19
from vllm.entrypoints.serve.disagg.serving import ServingTokens
20
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
21
from vllm.logprobs import Logprob
@@ -164,6 +167,36 @@ def _parse_sse_chunks(chunks: list[str]) -> list[Any]:
167
return parsed
168
169
170
+@pytest.mark.asyncio
171
+async def test_serve_tokens_skips_mm_cache_for_remote_engine_execution():
172
+ engine = _mock_engine()
173
+
174
+ async def mock_generate(*args, **kwargs):
175
+ yield _make_request_output(
176
+ "req-1", token_ids=[10], finish_reason="stop", finished=True
177
+ )
178
+
179
+ engine.generate = MagicMock(side_effect=mock_generate)
180
+ serving = _build_serving_tokens(engine)
181
+
182
+ request = GenerateRequest(
183
+ token_ids=[1, 2, 3],
184
+ sampling_params=SamplingParams(max_tokens=1),
185
+ model=MODEL_NAME,
186
+ stream=False,
187
+ )
188
+
189
+ response = await serving.serve_tokens(request)
190
+
191
+ assert isinstance(response, GenerateResponse)
192
+ assert (
193
+ serving.openai_serving_render.preprocess_completion.call_args.kwargs[
194
+ "skip_mm_cache"
195
+ ]
196
+ is True
197
+ )
198
+
199
+
200
@pytest.mark.asyncio
201
async def test_stream_basic():
202
"""Streaming returns SSE chunks with correct token_ids and ends with [DONE]."""
tests/entrypoints/serve/tokenize/test_serving_tokenization.py
new
+140
@@ -0,0 +1,140 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+from dataclasses import dataclass, field
5
+from typing import Any
6
+from unittest.mock import AsyncMock, MagicMock
7
+
8
+import pytest
9
+
10
+from vllm.config.multimodal import MultiModalConfig
11
+from vllm.entrypoints.openai.models.protocol import BaseModelPath
12
+from vllm.entrypoints.openai.models.serving import OpenAIServingModels
13
+from vllm.entrypoints.serve.render.serving import OpenAIServingRender
14
+from vllm.entrypoints.serve.tokenize.protocol import (
15
+ TokenizeChatRequest,
16
+ TokenizeCompletionRequest,
17
+)
18
+from vllm.entrypoints.serve.tokenize.serving import OpenAIServingTokenization
19
+from vllm.v1.engine.async_llm import AsyncLLM
20
+
21
+MODEL_NAME = "openai-community/gpt2"
22
+BASE_MODEL_PATHS = [
23
+ BaseModelPath(name=MODEL_NAME, model_path=MODEL_NAME),
24
+]
25
+
26
+
27
+@dataclass
28
+class MockHFConfig:
29
+ model_type: str = "any"
30
+
31
+
32
+@dataclass
33
+class MockModelConfig:
34
+ task = "generate"
35
+ runner_type = "generate"
36
+ model = MODEL_NAME
37
+ tokenizer = MODEL_NAME
38
+ trust_remote_code = False
39
+ tokenizer_mode = "auto"
40
+ max_model_len = 100
41
+ tokenizer_revision = None
42
+ multimodal_config = MultiModalConfig()
43
+ hf_config = MockHFConfig()
44
+ hf_text_config = MockHFConfig()
45
+ logits_processors: list[str] | None = None
46
+ diff_sampling_param: dict | None = None
47
+ allowed_local_media_path: str = ""
48
+ allowed_media_domains: list[str] | None = None
49
+ encoder_config = None
50
+ generation_config: str = "auto"
51
+ media_io_kwargs: dict[str, dict[str, Any]] = field(default_factory=dict)
52
+ skip_tokenizer_init = False
53
+ is_encoder_decoder: bool = False
54
+ is_multimodal_model: bool = False
55
+ renderer_num_workers: int = 1
56
+
57
+ def get_diff_sampling_param(self):
58
+ return self.diff_sampling_param or {}
59
+
60
+
61
+def _build_serving_tokenization(engine: AsyncLLM) -> OpenAIServingTokenization:
62
+ models = OpenAIServingModels(
63
+ engine_client=engine,
64
+ base_model_paths=BASE_MODEL_PATHS,
65
+ )
66
+ serving_render = OpenAIServingRender(
67
+ model_config=engine.model_config,
68
+ renderer=engine.renderer,
69
+ model_registry=models.registry,
70
+ request_logger=None,
71
+ chat_template=None,
72
+ chat_template_content_format="auto",
73
+ )
74
+ return OpenAIServingTokenization(
75
+ engine,
76
+ models,
77
+ openai_serving_render=serving_render,
78
+ request_logger=None,
79
+ chat_template=None,
80
+ chat_template_content_format="auto",
81
+ )
82
+
83
+
84
+@pytest.mark.asyncio
85
+async def test_tokenize_chat_skips_mm_cache_for_renderer_only_path():
86
+ mock_engine = MagicMock(spec=AsyncLLM)
87
+ mock_engine.errored = False
88
+ mock_engine.model_config = MockModelConfig()
89
+ mock_engine.input_processor = MagicMock()
90
+ mock_engine.renderer = MagicMock()
91
+
92
+ serving = _build_serving_tokenization(mock_engine)
93
+ serving.openai_serving_render.preprocess_chat = AsyncMock(
94
+ return_value=(
95
+ [{"role": "user", "content": "Test"}],
96
+ [{"prompt_token_ids": [1, 2, 3]}],
97
+ )
98
+ )
99
+
100
+ request = TokenizeChatRequest(
101
+ model=MODEL_NAME,
102
+ messages=[{"role": "user", "content": "Test prompt"}],
103
+ )
104
+
105
+ response = await serving.create_tokenize(request, MagicMock(headers={}))
106
+
107
+ assert response.tokens == [1, 2, 3]
108
+ assert (
109
+ serving.openai_serving_render.preprocess_chat.call_args.kwargs["skip_mm_cache"]
110
+ is True
111
+ )
112
+
113
+
114
+@pytest.mark.asyncio
115
+async def test_tokenize_completion_skips_mm_cache_for_renderer_only_path():
116
+ mock_engine = MagicMock(spec=AsyncLLM)
117
+ mock_engine.errored = False
118
+ mock_engine.model_config = MockModelConfig()
119
+ mock_engine.input_processor = MagicMock()
120
+ mock_engine.renderer = MagicMock()
121
+
122
+ serving = _build_serving_tokenization(mock_engine)
123
+ serving.openai_serving_render.preprocess_completion = AsyncMock(
124
+ return_value=[{"prompt_token_ids": [1, 2, 3]}]
125
+ )
126
+
127
+ request = TokenizeCompletionRequest(
128
+ model=MODEL_NAME,
129
+ prompt="Test prompt",
130
+ )
131
+
132
+ response = await serving.create_tokenize(request, MagicMock(headers={}))
133
+
134
+ assert response.tokens == [1, 2, 3]
135
+ assert (
136
+ serving.openai_serving_render.preprocess_completion.call_args.kwargs[
137
+ "skip_mm_cache"
138
+ ]
139
+ is True
140
+ )
tests/renderers/test_warmup.py
+24
@@ -34,6 +34,14 @@ def _make_renderer_mock(mm_limits: dict[str, int]) -> MagicMock:
34
mm_processor = MagicMock()
35
mm_processor.info.allowed_mm_limits = mm_limits
36
renderer.mm_processor = mm_processor
37
+ renderer._readonly_mm_processor = None
38
+ renderer._warmup_mm_processor = BaseRenderer._warmup_mm_processor.__get__(
39
+ renderer, BaseRenderer
40
+ )
41
+ renderer._clear_processor_cache = BaseRenderer._clear_processor_cache
42
+ renderer.clear_mm_cache = MagicMock()
43
+ renderer.model_config.max_model_len = 128
44
+ renderer.model_config.get_multimodal_config.return_value.limit_per_prompt = {}
45
46
return renderer
47
@@ -109,3 +117,19 @@ class TestMmWarmupSkippedWhenNoProcessor:
117
BaseRenderer.warmup(renderer, ChatParams())
118
119
renderer.model_config.get_multimodal_config.assert_not_called()
120
+
121
+
122
+class TestReadonlyMmWarmup:
123
+ """Readonly MM processor warmup must mirror the render path behavior."""
124
+
125
+ def test_readonly_processor_apply_called_and_cache_cleared(self):
126
+ renderer = _make_renderer_mock({"image": 1})
127
+ readonly_mm_processor = MagicMock()
128
+ readonly_mm_processor.info.allowed_mm_limits = {"image": 1}
129
+ renderer._readonly_mm_processor = readonly_mm_processor
130
+
131
+ with patch("vllm.multimodal.processing.TimingContext", autospec=True):
132
+ BaseRenderer.warmup(renderer, ChatParams())
133
+
134
+ readonly_mm_processor.apply.assert_called_once()
135
+ readonly_mm_processor.cache.clear_cache.assert_called_once()
vllm/entrypoints/serve/render/serving.py
+8
-4
@@ -136,7 +136,7 @@ class OpenAIServingRender:
136
"Beam search is not supported by the render endpoint"
137
)
138
139
- result = await self.render_chat(request)
139
+ result = await self.render_chat(request, skip_mm_cache=True)
140
if isinstance(result, ErrorResponse):
141
return result
142
@@ -184,6 +184,8 @@ class OpenAIServingRender:
184
async def render_chat(
185
self,
186
request: ChatCompletionRequest,
187
+ *,
188
+ skip_mm_cache: bool = False,
189
) -> tuple[list[ConversationMessage], list[EngineInput]] | ErrorResponse:
190
"""Core preprocessing logic for chat requests (no model/engine check).
191
@@ -252,7 +254,7 @@ class OpenAIServingRender:
254
default_template_kwargs=self.default_chat_template_kwargs,
255
tool_dicts=tool_dicts,
256
tool_parser=tool_parser,
255
- skip_mm_cache=True,
257
+ skip_mm_cache=skip_mm_cache,
258
reasoning_parser=self.reasoning_parser,
259
)
260
else:
@@ -276,7 +278,7 @@ class OpenAIServingRender:
278
error_check_ret = await self._check_model(request)
279
if error_check_ret is not None:
280
return error_check_ret
279
- result = await self.render_completion(request)
281
+ result = await self.render_completion(request, skip_mm_cache=True)
282
if isinstance(result, ErrorResponse):
283
return result
284
generate_requests: list[GenerateRequest] = []
@@ -322,6 +324,8 @@ class OpenAIServingRender:
324
async def render_completion(
325
self,
326
request: CompletionRequest,
327
+ *,
328
+ skip_mm_cache: bool = False,
329
) -> list[EngineInput] | ErrorResponse:
330
"""Core preprocessing logic for completion requests (no model/engine check).
331
@@ -344,7 +348,7 @@ class OpenAIServingRender:
348
request,
349
prompt_input=request.prompt,
350
prompt_embeds=request.prompt_embeds,
347
- skip_mm_cache=True,
351
+ skip_mm_cache=skip_mm_cache,
352
)
353
354
return engine_inputs
vllm/renderers/base.py
+49
-21
@@ -198,6 +198,40 @@ class BaseRenderer(ABC, Generic[_T]):
198
if self._mm_cache_stats is not None:
199
self._mm_cache_stats.reset = True
200
201
+ @staticmethod
202
+ def _clear_processor_cache(
203
+ processor: "BaseMultiModalProcessor | None",
204
+ ) -> None:
205
+ if processor is None:
206
+ return
207
+
208
+ processor_cache = processor.cache
209
+ if processor_cache is not None:
210
+ processor_cache.clear_cache()
211
+
212
+ def _warmup_mm_processor(
213
+ self,
214
+ processor: "BaseMultiModalProcessor",
215
+ *,
216
+ log_prefix: str,
217
+ ) -> None:
218
+ from vllm.multimodal.processing import TimingContext
219
+
220
+ model_config = self.model_config
221
+ mm_config = model_config.get_multimodal_config()
222
+ mm_limits = {k: v for k, v in processor.info.allowed_mm_limits.items() if v > 0}
223
+
224
+ start_time = time.perf_counter()
225
+ processor_inputs = processor.dummy_inputs.get_dummy_processor_inputs(
226
+ seq_len=model_config.max_model_len,
227
+ mm_counts=dict.fromkeys(mm_limits, 1),
228
+ mm_options=mm_config.limit_per_prompt,
229
+ )
230
+ _ = processor.apply(processor_inputs, timing_ctx=TimingContext(enabled=False))
231
+
232
+ elapsed = time.perf_counter() - start_time
233
+ logger.info("%s warmup completed in %.3fs", log_prefix, elapsed)
234
+
235
def warmup(self, chat_params: ChatParams) -> None:
236
"""
237
Warm up this renderer to avoid first-request latency.
@@ -221,35 +255,29 @@ class BaseRenderer(ABC, Generic[_T]):
255
logger.warning("Chat template warmup failed", exc_info=True)
256
257
if self.mm_processor:
224
- from vllm.multimodal.processing import TimingContext
225
-
226
- model_config = self.model_config
227
- mm_config = model_config.get_multimodal_config()
228
- processor = self.mm_processor
229
- mm_limits = {
230
- k: v for k, v in processor.info.allowed_mm_limits.items() if v > 0
231
- }
232
-
258
try:
259
logger.debug("Warming up multi-modal processing...")
235
- start_time = time.perf_counter()
236
-
237
- processor_inputs = processor.dummy_inputs.get_dummy_processor_inputs(
238
- seq_len=model_config.max_model_len,
239
- mm_counts=dict.fromkeys(mm_limits, 1),
240
- mm_options=mm_config.limit_per_prompt,
241
- )
242
- _ = processor.apply(
243
- processor_inputs, timing_ctx=TimingContext(enabled=False)
260
+ self._warmup_mm_processor(
261
+ self.mm_processor,
262
+ log_prefix="Multi-modal",
263
)
245
-
246
- elapsed = time.perf_counter() - start_time
247
- logger.info("Multi-modal warmup completed in %.3fs", elapsed)
264
except Exception:
265
logger.warning("Multi-modal warmup failed")
266
finally:
267
self.clear_mm_cache()
268
269
+ if self._readonly_mm_processor is not None:
270
+ try:
271
+ logger.debug("Warming up readonly multi-modal processing...")
272
+ self._warmup_mm_processor(
273
+ self._readonly_mm_processor,
274
+ log_prefix="Readonly multi-modal",
275
+ )
276
+ except Exception:
277
+ logger.warning("Readonly multi-modal warmup failed")
278
+ finally:
279
+ self._clear_processor_cache(self._readonly_mm_processor)
280
+
281
async def clear_mm_cache_async(self) -> None:
282
"""Serialize clear_mm_cache through the shared executor to avoid
283
races with concurrent process_inputs on the mm_processor_cache."""