A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 1, 18:45 UTC
[CI] Stabilize the multi-audio OpenAI server path (#44051)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Andreas Karatzas committed
Jun 7, 2026 at 02:54 UTC
f0f6805d8a0c38d676414f348f36ffe519dd895d
3 files changed
+62
-59
vllm/entrypoints/chat_utils.py
+58
-58
@@ -406,6 +406,7 @@ ModalityStr = Literal[
406
"prompt_embeds",
407
]
408
_T = TypeVar("_T")
409
+_AsyncMultiModalItem: TypeAlias = Callable[[], Awaitable[tuple[object, str | None]]]
410
411
412
# Backward compatibility for single item input
@@ -596,10 +597,27 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
597
or model-specific placeholder logic. The corresponding placeholder string is
598
managed by the parser via `_add_placeholder`, so we return None here.
599
"""
599
- if modality == "prompt_embeds":
600
+ add_info = self._validate_add(modality)
601
+ if add_info is None:
602
self._items_by_modality["prompt_embeds"].append(item)
603
return None
604
605
+ input_modality, original_modality, use_vision_chunk, num_items = add_info
606
+
607
+ # Track original modality for vision_chunk items
608
+ if use_vision_chunk:
609
+ self._items_by_modality[input_modality].append(item) # type: ignore
610
+ self._modality_order["vision_chunk"].append(original_modality)
611
+ else:
612
+ self._items_by_modality[original_modality].append(item)
613
+
614
+ return self.model_cls.get_placeholder_str(modality, num_items)
615
+
616
+ def _validate_add(self, modality: ModalityStr) -> tuple[str, str, bool, int] | None:
617
+ """Validate that one more item of the modality can be tracked."""
618
+ if modality == "prompt_embeds":
619
+ return None
620
+
621
input_modality = modality.replace("_embeds", "")
622
original_modality = modality
623
use_vision_chunk = (
@@ -630,14 +648,7 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
648
else:
649
self.mm_processor.info.validate_num_items(input_modality, num_items)
650
633
- # Track original modality for vision_chunk items
634
- if use_vision_chunk:
635
- self._items_by_modality[input_modality].append(item) # type: ignore
636
- self._modality_order["vision_chunk"].append(original_modality)
637
- else:
638
- self._items_by_modality[original_modality].append(item)
639
-
640
- return self.model_cls.get_placeholder_str(modality, num_items)
651
+ return input_modality, original_modality, use_vision_chunk, num_items
652
653
@abstractmethod
654
def create_parser(
@@ -803,9 +814,7 @@ class MultiModalItemTracker(BaseMultiModalItemTracker[tuple[object, str | None]]
814
return MultiModalContentParser(self, mm_processor_kwargs=mm_processor_kwargs)
815
816
806
-class AsyncMultiModalItemTracker(
807
- BaseMultiModalItemTracker[Awaitable[tuple[object, str | None]]]
808
-):
817
+class AsyncMultiModalItemTracker(BaseMultiModalItemTracker[_AsyncMultiModalItem]):
818
async def resolve_items(
819
self,
820
) -> tuple[MultiModalDataDict | None, MultiModalUUIDDict | None]:
@@ -813,8 +822,8 @@ class AsyncMultiModalItemTracker(
822
return None, None
823
824
resolved_items_by_modality = {
816
- modality: await asyncio.gather(*coros)
817
- for modality, coros in self._items_by_modality.items()
825
+ modality: await asyncio.gather(*(item() for item in items))
826
+ for modality, items in self._items_by_modality.items()
827
}
828
829
mm_processor = (
@@ -1074,6 +1083,9 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
1083
def model_config(self) -> ModelConfig:
1084
return self._tracker.model_config
1085
1086
+ async def _item_with_uuid_async(self, item: object, uuid: str | None):
1087
+ return item, uuid
1088
+
1089
@override
1090
def parse_prompt_embeds(self, data: str) -> None:
1091
"""Schedule async prompt embeds decode and store the coroutine in the tracker.
@@ -1085,8 +1097,9 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
1097
if not self.model_config.enable_prompt_embeds:
1098
raise ValueError(_ENABLE_PROMPT_EMBEDS_ERROR)
1099
1088
- coro = self._load_prompt_embeds_async(data.encode())
1089
- self._tracker.add("prompt_embeds", coro)
1100
+ self._tracker.add(
1101
+ "prompt_embeds", partial(self._load_prompt_embeds_async, data.encode())
1102
+ )
1103
self._add_placeholder("prompt_embeds", PROMPT_EMBEDS_PLACEHOLDER_TOKEN)
1104
1105
async def _load_prompt_embeds_async(
@@ -1104,9 +1117,9 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
1117
return image, uuid
1118
1119
def parse_image(self, image_url: str | None, uuid: str | None = None) -> None:
1107
- coro = self._image_with_uuid_async(image_url, uuid)
1108
-
1109
- placeholder = self._tracker.add("image", coro)
1120
+ placeholder = self._tracker.add(
1121
+ "image", partial(self._image_with_uuid_async, image_url, uuid)
1122
+ )
1123
self._add_placeholder("image", placeholder)
1124
1125
def parse_image_embeds(
@@ -1120,25 +1133,20 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
1133
"You must set `--enable-mm-embeds` to input `image_embeds`"
1134
)
1135
1123
- future = asyncio.Future[
1124
- tuple[torch.Tensor | dict[str, torch.Tensor] | None, str | None]
1125
- ]()
1126
-
1136
if isinstance(image_embeds, dict):
1137
embeds = {
1138
k: self._connector.fetch_image_embedding(v)
1139
for k, v in image_embeds.items()
1140
}
1132
- future.set_result((embeds, uuid))
1133
-
1134
- if isinstance(image_embeds, str):
1141
+ elif isinstance(image_embeds, str):
1142
embedding = self._connector.fetch_image_embedding(image_embeds)
1136
- future.set_result((embedding, uuid))
1137
-
1138
- if image_embeds is None:
1139
- future.set_result((None, uuid))
1143
+ embeds = embedding
1144
+ else:
1145
+ embeds = None
1146
1141
- placeholder = self._tracker.add("image_embeds", future)
1147
+ placeholder = self._tracker.add(
1148
+ "image_embeds", partial(self._item_with_uuid_async, embeds, uuid)
1149
+ )
1150
self._add_placeholder("image", placeholder)
1151
1152
def parse_audio_embeds(
@@ -1152,25 +1160,20 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
1160
"You must set `--enable-mm-embeds` to input `audio_embeds`"
1161
)
1162
1155
- future = asyncio.Future[
1156
- tuple[torch.Tensor | dict[str, torch.Tensor] | None, str | None]
1157
- ]()
1158
-
1163
if isinstance(audio_embeds, dict):
1164
embeds = {
1165
k: self._connector.fetch_audio_embedding(v)
1166
for k, v in audio_embeds.items()
1167
}
1164
- future.set_result((embeds, uuid))
1165
-
1166
- if isinstance(audio_embeds, str):
1168
+ elif isinstance(audio_embeds, str):
1169
embedding = self._connector.fetch_audio_embedding(audio_embeds)
1168
- future.set_result((embedding, uuid))
1169
-
1170
- if audio_embeds is None:
1171
- future.set_result((None, uuid))
1170
+ embeds = embedding
1171
+ else:
1172
+ embeds = None
1173
1173
- placeholder = self._tracker.add("audio_embeds", future)
1174
+ placeholder = self._tracker.add(
1175
+ "audio_embeds", partial(self._item_with_uuid_async, embeds, uuid)
1176
+ )
1177
self._add_placeholder("audio", placeholder)
1178
1179
def parse_image_pil(
@@ -1178,13 +1181,9 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
1181
image_pil: Image.Image | None,
1182
uuid: str | None = None,
1183
) -> None:
1181
- future = asyncio.Future[tuple[Image.Image | None, str | None]]()
1182
- if image_pil:
1183
- future.set_result((image_pil, uuid))
1184
- else:
1185
- future.set_result((None, uuid))
1186
-
1187
- placeholder = self._tracker.add("image", future)
1184
+ placeholder = self._tracker.add(
1185
+ "image", partial(self._item_with_uuid_async, image_pil, uuid)
1186
+ )
1187
self._add_placeholder("image", placeholder)
1188
1189
async def _audio_with_uuid_async(self, audio_url: str | None, uuid: str | None):
@@ -1194,9 +1193,9 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
1193
return audio, uuid
1194
1195
def parse_audio(self, audio_url: str | None, uuid: str | None = None) -> None:
1197
- coro = self._audio_with_uuid_async(audio_url, uuid)
1198
-
1199
- placeholder = self._tracker.add("audio", coro)
1196
+ placeholder = self._tracker.add(
1197
+ "audio", partial(self._audio_with_uuid_async, audio_url, uuid)
1198
+ )
1199
self._add_placeholder("audio", placeholder)
1200
1201
def parse_input_audio(
@@ -1227,9 +1226,9 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
1226
return video, uuid
1227
1228
def parse_video(self, video_url: str | None, uuid: str | None = None) -> None:
1230
- coro = self._video_with_uuid_async(video_url, uuid)
1231
-
1232
- placeholder = self._tracker.add("video", coro)
1229
+ placeholder = self._tracker.add(
1230
+ "video", partial(self._video_with_uuid_async, video_url, uuid)
1231
+ )
1232
self._add_placeholder("video", placeholder)
1233
1234
# Extract audio from video if use_audio_in_video is True
@@ -1238,8 +1237,9 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
1237
and self._mm_processor_kwargs
1238
and self._mm_processor_kwargs.get("use_audio_in_video", False)
1239
):
1241
- audio_coro = self._audio_with_uuid_async(video_url, uuid)
1242
- audio_placeholder = self._tracker.add("audio", audio_coro)
1240
+ audio_placeholder = self._tracker.add(
1241
+ "audio", partial(self._audio_with_uuid_async, video_url, uuid)
1242
+ )
1243
self._add_placeholder("audio", audio_placeholder)
1244
1245
vllm/entrypoints/openai/api_server.py
+2
@@ -52,6 +52,7 @@ from vllm.entrypoints.serve.utils.server_utils import (
52
log_response,
53
validation_exception_handler,
54
)
55
+from vllm.exceptions import VLLMValidationError
56
from vllm.logger import init_logger
57
from vllm.reasoning import ReasoningParserManager
58
from vllm.tasks import POOLING_TASKS, SupportedTask
@@ -250,6 +251,7 @@ def build_app(
251
app.exception_handler(EngineGenerateError)(engine_error_handler)
252
app.exception_handler(EngineDeadError)(engine_error_handler)
253
app.exception_handler(GenerationError)(generation_error_handler)
254
+ app.exception_handler(VLLMValidationError)(exception_handler)
255
app.exception_handler(Exception)(exception_handler)
256
257
# Ensure --api-key option from CLI takes precedence over VLLM_API_KEY
vllm/multimodal/processing/context.py
+2
-1
@@ -11,6 +11,7 @@ from typing import TYPE_CHECKING, Any, overload
11
import torch
12
from typing_extensions import TypeVar
13
14
+from vllm.exceptions import VLLMValidationError
15
from vllm.inputs import MultiModalDataDict
16
from vllm.logger import init_logger
17
from vllm.multimodal.parse import (
@@ -424,7 +425,7 @@ class BaseProcessingInfo:
425
if num_items <= supported_limit:
426
msg += " Set `--limit-mm-per-prompt` to increase this limit."
427
427
- raise ValueError(msg)
428
+ raise VLLMValidationError(msg, parameter=modality)
429
430
def parse_mm_data(
431
self,