[Core] Offload raw-prompt preprocessing to renderer thread pool in AsyncLLM (#49608)
Signed-off-by: almogtavor <almogtavor@gmail.com>
Almog Tavor committed
Aug 1, 2026 at 18:10 UTC
39f55ffdaacbf4f2ac06e395c3f9c6e55a54dbd0
2 files changed
+37
-12
vllm/v1/engine/async_llm.py
+29
-12
@@ -349,18 +349,35 @@ class AsyncLLM(EngineClient):
349
"latter will be used, and the former will be ignored."
350
)
351
else:
352
- request = self.input_processor.process_inputs(
353
- request_id,
354
- prompt,
355
- params,
356
- supported_tasks=await self.get_supported_tasks(),
357
- arrival_time=arrival_time,
358
- lora_request=lora_request,
359
- tokenization_kwargs=tokenization_kwargs,
360
- trace_headers=trace_headers,
361
- priority=priority,
362
- data_parallel_rank=data_parallel_rank,
363
- )
352
+ if isinstance(prompt, dict) and "type" in prompt:
353
+ # Rendered EngineInput; no blocking preprocessing needed.
354
+ request = self.input_processor.process_inputs(
355
+ request_id,
356
+ prompt,
357
+ params,
358
+ supported_tasks=await self.get_supported_tasks(),
359
+ arrival_time=arrival_time,
360
+ lora_request=lora_request,
361
+ tokenization_kwargs=tokenization_kwargs,
362
+ trace_headers=trace_headers,
363
+ priority=priority,
364
+ data_parallel_rank=data_parallel_rank,
365
+ )
366
+ else:
367
+ # Raw prompts require tokenization and possibly multimodal
368
+ # processing, which must not block the event loop.
369
+ request = await self.input_processor.process_inputs_async(
370
+ request_id,
371
+ prompt,
372
+ params,
373
+ supported_tasks=await self.get_supported_tasks(),
374
+ arrival_time=arrival_time,
375
+ lora_request=lora_request,
376
+ tokenization_kwargs=tokenization_kwargs,
377
+ trace_headers=trace_headers,
378
+ priority=priority,
379
+ data_parallel_rank=data_parallel_rank,
380
+ )
381
prompt_text, _, _ = extract_prompt_components(self.model_config, prompt)
382
383
if reasoning_ended is not None:
vllm/v1/engine/input_processor.py
+8
@@ -28,6 +28,7 @@ from vllm.sampling_params import SamplingParams
28
from vllm.tasks import GENERATION_TASKS, POOLING_TASKS, SupportedTask
29
from vllm.tokenizers import TokenizerLike
30
from vllm.utils import length_from_prompt_token_ids_or_embeds, random_uuid
31
+from vllm.utils.async_utils import make_async
32
from vllm.utils.jsontree import json_iter_leaves
33
from vllm.v1.engine import EngineCoreRequest
34
@@ -73,6 +74,13 @@ class InputProcessor:
74
mm_registry=mm_registry,
75
)
76
77
+ # Raw-prompt preprocessing (tokenization and multimodal processing)
78
+ # is blocking, so async callers should run it on the renderer's
79
+ # thread pool to keep their event loop responsive.
80
+ self.process_inputs_async = make_async(
81
+ self.process_inputs, executor=self.renderer._executor
82
+ )
83
+
84
@property
85
def tokenizer(self) -> TokenizerLike | None:
86
return self.renderer.tokenizer