A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 2, 09:45 UTC
offload prompt_embeds decode in render_prompts_async to avoid blocking (#43792)
Signed-off-by: Gagan Dhakrey <gagandhakrey@gmail.com>
Gagan Dhakrey committed
May 30, 2026 at 07:06 UTC
1e2ce5d11a9136f03823663a299479cd1cbbacfc
1 file changed
+21
-1
vllm/renderers/base.py
+21
-1
@@ -104,6 +104,9 @@ class BaseRenderer(ABC, Generic[_T]):
104
self._process_multimodal_async = make_async(
105
self._process_multimodal, executor=self._mm_executor
106
)
107
+ self._safe_load_prompt_embeds_async = make_async(
108
+ safe_load_prompt_embeds, executor=self._executor
109
+ )
110
if mm_registry.supports_multimodal_inputs(config.model_config):
111
mm_processor_cache = mm_registry.processor_cache_from_config(config)
112
@@ -376,11 +379,28 @@ class BaseRenderer(ABC, Generic[_T]):
379
380
return [self.render_prompt(prompt) for prompt in prompts]
381
382
+ async def _render_prompt_async(
383
+ self,
384
+ prompt: DictPrompt | bytes,
385
+ ) -> DictPrompt:
386
+ if isinstance(prompt, bytes):
387
+ embeds = await self._safe_load_prompt_embeds_async(
388
+ self.model_config, prompt
389
+ )
390
+ return EmbedsPrompt(prompt_embeds=embeds)
391
+
392
+ return prompt
393
+
394
async def render_prompts_async(
395
self,
396
prompts: Sequence[DictPrompt | bytes],
397
) -> list[DictPrompt]:
383
- return self.render_prompts(prompts)
398
+ if len(prompts) == 0:
399
+ raise ValueError("You must pass at least one prompt")
400
+
401
+ return await asyncio.gather(
402
+ *(self._render_prompt_async(prompt) for prompt in prompts)
403
+ )
404
405
@abstractmethod
406
def render_messages(