[Perf] Defer MM embeds loading off the event loop (#49477)
Signed-off-by: Guan-Ming (Wesley) Chiu <105915352+guan404ming@users.noreply.github.com>
Guan-Ming Chiu committed
Jul 23, 2026 at 21:58 UTC
80c7683923795e9c2e8929fb8b766aecb0a63447
2 files changed
+68
-22
vllm/entrypoints/chat_utils.py
+40
-22
@@ -1162,21 +1162,30 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
1162
parameter="image_embeds",
1163
)
1164
1165
+ placeholder = self._tracker.add(
1166
+ "image_embeds",
1167
+ partial(self._image_embeds_with_uuid_async, image_embeds, uuid),
1168
+ )
1169
+ self._add_placeholder("image", placeholder)
1170
+
1171
+ async def _image_embeds_with_uuid_async(
1172
+ self,
1173
+ image_embeds: str | dict[str, str] | None,
1174
+ uuid: str | None,
1175
+ ):
1176
if isinstance(image_embeds, dict):
1166
- embeds = {
1167
- k: self._connector.fetch_image_embedding(v)
1168
- for k, v in image_embeds.items()
1169
- }
1177
+ tensors = await asyncio.gather(
1178
+ *(
1179
+ self._connector.fetch_image_embedding_async(v)
1180
+ for v in image_embeds.values()
1181
+ )
1182
+ )
1183
+ embeds = dict(zip(image_embeds, tensors))
1184
elif isinstance(image_embeds, str):
1171
- embedding = self._connector.fetch_image_embedding(image_embeds)
1172
- embeds = embedding
1185
+ embeds = await self._connector.fetch_image_embedding_async(image_embeds)
1186
else:
1187
embeds = None
1175
-
1176
- placeholder = self._tracker.add(
1177
- "image_embeds", partial(self._item_with_uuid_async, embeds, uuid)
1178
- )
1179
- self._add_placeholder("image", placeholder)
1188
+ return embeds, uuid
1189
1190
def parse_audio_embeds(
1191
self,
@@ -1190,21 +1199,30 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
1199
parameter="audio_embeds",
1200
)
1201
1202
+ placeholder = self._tracker.add(
1203
+ "audio_embeds",
1204
+ partial(self._audio_embeds_with_uuid_async, audio_embeds, uuid),
1205
+ )
1206
+ self._add_placeholder("audio", placeholder)
1207
+
1208
+ async def _audio_embeds_with_uuid_async(
1209
+ self,
1210
+ audio_embeds: str | dict[str, str] | None,
1211
+ uuid: str | None,
1212
+ ):
1213
if isinstance(audio_embeds, dict):
1194
- embeds = {
1195
- k: self._connector.fetch_audio_embedding(v)
1196
- for k, v in audio_embeds.items()
1197
- }
1214
+ tensors = await asyncio.gather(
1215
+ *(
1216
+ self._connector.fetch_audio_embedding_async(v)
1217
+ for v in audio_embeds.values()
1218
+ )
1219
+ )
1220
+ embeds = dict(zip(audio_embeds, tensors))
1221
elif isinstance(audio_embeds, str):
1199
- embedding = self._connector.fetch_audio_embedding(audio_embeds)
1200
- embeds = embedding
1222
+ embeds = await self._connector.fetch_audio_embedding_async(audio_embeds)
1223
else:
1224
embeds = None
1203
-
1204
- placeholder = self._tracker.add(
1205
- "audio_embeds", partial(self._item_with_uuid_async, embeds, uuid)
1206
- )
1207
- self._add_placeholder("audio", placeholder)
1225
+ return embeds, uuid
1226
1227
def parse_image_pil(
1228
self,
vllm/multimodal/media/connector.py
+28
@@ -593,6 +593,20 @@ class MediaConnector:
593
594
return image_embedding_io.load_base64("", data)
595
596
+ async def fetch_image_embedding_async(
597
+ self,
598
+ data: str,
599
+ ) -> torch.Tensor:
600
+ """
601
+ Asynchronously load image embedding from a URL.
602
+ """
603
+ image_embedding_io = ImageEmbeddingMediaIO()
604
+ loop = asyncio.get_running_loop()
605
+
606
+ return await loop.run_in_executor(
607
+ global_thread_pool, image_embedding_io.load_base64, "", data
608
+ )
609
+
610
def fetch_audio_embedding(
611
self,
612
data: str,
@@ -603,3 +617,17 @@ class MediaConnector:
617
audio_embedding_io = AudioEmbeddingMediaIO()
618
619
return audio_embedding_io.load_base64("", data)
620
+
621
+ async def fetch_audio_embedding_async(
622
+ self,
623
+ data: str,
624
+ ) -> torch.Tensor:
625
+ """
626
+ Asynchronously load audio embedding from a URL.
627
+ """
628
+ audio_embedding_io = AudioEmbeddingMediaIO()
629
+ loop = asyncio.get_running_loop()
630
+
631
+ return await loop.run_in_executor(
632
+ global_thread_pool, audio_embedding_io.load_base64, "", data
633
+ )