@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 555ea65e8c

[Frontend] Add video embeds input support (#54242)

Signed-off-by: Isotr0py <Isotr0py@outlook.com>

Isotr0py committed Aug 31, 2026 at 11:20 UTC 555ea65e8c41732e62eed93915f7ac6bf9497f6c

8 files changed +461 -24

examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py

+26 -15

@@ -10,7 +10,7 @@ clusters:

10 • decode (language-model inference)

12 For MM input we:

13 - 1. Extract *every* image/audio item.

13 + 1. Extract *every* image/audio/video item.

14 2. Fire N concurrent requests to the encoder cluster

15 (one request per item, with **all text removed**).

16 3. Wait for all of them to succeed.

@@ -61,7 +61,16 @@ encoder_rr_lock = asyncio.Lock()

61 ###############################################################################

64 -MM_TYPES = {"image_url", "audio_url", "input_audio"}

64 +MM_TYPES = {"image_url", "audio_url", "input_audio", "video_url"}

65 +

66 +# The embeds content type each MM item is rewritten to once the encoder has

67 +# published its embedding out of band.

68 +EMBEDS_TYPES = {

69 + "image_url": "image_embeds",

70 + "audio_url": "audio_embeds",

71 + "input_audio": "audio_embeds",

72 + "video_url": "video_embeds",

73 +}

76 def encoder_rr_assignment(

@@ -94,7 +103,9 @@ def content_uuid(item: dict) -> str:

103 the unmodified path, which hashes the content, would keep it. That asymmetry

104 silently biases any comparison between the two.

105 """

97 - url = (item.get("image_url") or item.get("audio_url") or {}).get("url") or ""

106 + url = (

107 + item.get("image_url") or item.get("audio_url") or item.get("video_url") or {}

108 + ).get("url") or ""

109 payload = url or json.dumps(item, sort_keys=True)

110 return hashlib.sha256(payload.encode()).hexdigest()

111

@@ -103,19 +114,22 @@ def _b64_tensor(values: list) -> str:

114 import torch

115

116 buf = io.BytesIO()

106 - grid = torch.tensor(values, dtype=torch.long)

107 - # Downstream stacks per item, so hand over a flat (t, h, w).

108 - torch.save(grid.reshape(-1)[:3], buf)

117 + flat = [v for item in values for v in (item if isinstance(item, list) else [item])]

118 + # Floats stay float64 so timestamp strings format exactly as the

119 + # encoder computed them.

120 + dtype = torch.float64 if any(isinstance(v, float) for v in flat) else None

121 + # Downstream stacks per item, so hand over a flat vector.

122 + torch.save(torch.tensor(flat, dtype=dtype), buf)

123 return base64.b64encode(buf.getvalue()).decode()

124

125

126 def rewrite_for_decode(req_data: dict, item_meta: dict[int, dict]) -> dict:

113 - """Replace each image item with a metadata-only reference for the decoder.

127 + """Replace each media item with a metadata-only reference for the decoder.

128

129 The decoder does not need the pixels: the encoder instance already produced

130 the embedding and published it through the EC connector under the same uuid.

131 Sending only the grid lets the decoder size the placeholder range without

118 - re-running the image transform.

132 + re-running the media transform.

133

134 `item_meta` holds what the encoder reported for each item (its cache key and

135 the grid its processor actually produced), so the grid is never re-derived

@@ -146,25 +160,22 @@ def rewrite_for_decode(req_data: dict, item_meta: dict[int, dict]) -> dict:

160 # processor cache); let the decoder process the media itself.

161 new_content.append(item)

162 continue

163 + embeds_type = EMBEDS_TYPES[item["type"]]

164 new_content.append(

150 - {

151 - "type": "image_embeds",

152 - "image_embeds": metadata,

153 - "uuid": item_uuid,

154 - }

165 + {"type": embeds_type, embeds_type: metadata, "uuid": item_uuid}

166 )

167 rewritten += 1

168 new_messages.append({**msg, "content": new_content})

169

170 if not rewritten:

171 return req_data

161 - logger.info("Rewrote %d image item(s) as metadata references", rewritten)

172 + logger.info("Rewrote %d media item(s) as metadata references", rewritten)

173 return {**req_data, "messages": new_messages}

174

175

176 def extract_mm_items(request_data: dict) -> list[dict]:

177 """

167 - Return *all* image/audio items that appear anywhere in `messages`.

178 + Return *all* image/audio/video items that appear anywhere in `messages`.

179

180 Each returned dict looks like:

181 { "type": "image_url", "image_url": {...} }

tests/entrypoints/unit_tests/test_chat_utils.py

+137

@@ -98,6 +98,16 @@ def qwen25omni_model_config_image_embeds():

98 )

100

101 +@pytest.fixture(scope="function")

102 +def qwen25omni_model_config_video_embeds():

103 + return ModelConfig(

104 + QWEN25OMNI_MODEL_ID,

105 + runner="generate",

106 + limit_mm_per_prompt={"video": 2},

107 + enable_mm_embeds=True,

108 + )

109 +

110 +

111 @pytest.fixture(scope="function")

112 def qwen2_audio_model_config():

113 return ModelConfig(

@@ -969,6 +979,133 @@ async def test_parse_chat_messages_audio_embeds_async(

979 _assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=[None])

980

981

982 +def test_parse_chat_messages_empty_video_embeds_with_uuid(

983 + qwen25omni_model_config_video_embeds,

984 +):

985 + """Test video_embeds with UUID (no actual embeds data)."""

986 + uuid = "test-video-uuid-123"

987 +

988 + conversation, mm_data, mm_uuids = parse_chat_messages(

989 + [

990 + {

991 + "role": "user",

992 + "content": [

993 + {"type": "text", "text": "Describe this video"},

994 + {"type": "video_embeds", "video_embeds": None, "uuid": uuid},

995 + ],

996 + }

997 + ],

998 + qwen25omni_model_config_video_embeds,

999 + content_format="string",

1000 + )

1001 +

1002 + # Should have video in mm_data as None (UUID provided)

1003 + assert mm_data is not None

1004 + assert "video" in mm_data

1005 + assert isinstance(mm_data["video"], list)

1006 + assert len(mm_data["video"]) == 1

1007 + assert mm_data["video"][0] is None

1008 +

1009 + # UUID should be recorded

1010 + _assert_mm_uuids(mm_uuids, 1, modality="video", expected_uuids=[uuid])

1011 +

1012 +

1013 +def test_parse_chat_messages_video_embeds_with_string(

1014 + qwen25omni_model_config_video_embeds,

1015 +):

1016 + """Test video_embeds with base64 string embedding data."""

1017 + hidden_size = qwen25omni_model_config_video_embeds.get_inputs_embeds_size()

1018 + video_embedding = torch.randn(1, 128, hidden_size)

1019 +

1020 + base64_video_embedding = tensor2base64(video_embedding)

1021 +

1022 + conversation, mm_data, mm_uuids = parse_chat_messages(

1023 + [

1024 + {

1025 + "role": "user",

1026 + "content": [

1027 + {"type": "text", "text": "Describe this video"},

1028 + {

1029 + "type": "video_embeds",

1030 + "video_embeds": base64_video_embedding,

1031 + },

1032 + ],

1033 + }

1034 + ],

1035 + qwen25omni_model_config_video_embeds,

1036 + content_format="string",

1037 + )

1038 +

1039 + # Should have video embedding in mm_data (single tensor, not a list)

1040 + assert mm_data is not None

1041 + assert "video" in mm_data

1042 + assert isinstance(mm_data["video"], torch.Tensor)

1043 + assert mm_data["video"].shape == video_embedding.shape

1044 + # No UUID provided

1045 + assert mm_uuids is not None

1046 + assert "video" in mm_uuids

1047 + _assert_mm_uuids(mm_uuids, 1, modality="video", expected_uuids=[None])

1048 +

1049 +

1050 +@pytest.mark.asyncio

1051 +async def test_parse_chat_messages_video_embeds_async(

1052 + qwen25omni_model_config_video_embeds,

1053 +):

1054 + """Test video_embeds with async futures."""

1055 + hidden_size = qwen25omni_model_config_video_embeds.get_inputs_embeds_size()

1056 + video_embedding = torch.randn(1, 128, hidden_size)

1057 +

1058 + base64_video_embedding = tensor2base64(video_embedding)

1059 +

1060 + conversation, mm_data, mm_uuids = await parse_chat_messages_async(

1061 + [

1062 + {

1063 + "role": "user",

1064 + "content": [

1065 + {"type": "text", "text": "Describe this video"},

1066 + {

1067 + "type": "video_embeds",

1068 + "video_embeds": base64_video_embedding,

1069 + },

1070 + ],

1071 + }

1072 + ],

1073 + qwen25omni_model_config_video_embeds,

1074 + content_format="string",

1075 + )

1076 +

1077 + # Should have video embedding in mm_data (single tensor, not a list)

1078 + assert mm_data is not None

1079 + assert "video" in mm_data

1080 + assert isinstance(mm_data["video"], torch.Tensor)

1081 + assert mm_data["video"].shape == video_embedding.shape

1082 + # No UUID provided

1083 + assert mm_uuids is not None

1084 + assert "video" in mm_uuids

1085 + _assert_mm_uuids(mm_uuids, 1, modality="video", expected_uuids=[None])

1086 +

1087 +

1088 +def test_parse_chat_messages_video_and_video_embeds_mixed(

1089 + qwen25omni_model_config_video_embeds,

1090 + video_url,

1091 +):

1092 + """Test that mixing raw video and video_embeds is rejected."""

1093 + with pytest.raises(VLLMValidationError, match="Mixing raw video"):

1094 + parse_chat_messages(

1095 + [

1096 + {

1097 + "role": "user",

1098 + "content": [

1099 + {"type": "video_url", "video_url": {"url": video_url}},

1100 + {"type": "video_embeds", "video_embeds": None, "uuid": "x"},

1101 + ],

1102 + }

1103 + ],

1104 + qwen25omni_model_config_video_embeds,

1105 + content_format="string",

1106 + )

1107 +

1108 +

1109 def test_parse_chat_messages_multiple_image_embeds(

1110 phi3v_model_config_image_embeds,

1111 ):

tests/models/multimodal/processing/test_qwen3_vl.py

+85

@@ -11,8 +11,10 @@ from typing import Any

11 import numpy as np

12 import pytest

14 +from vllm.config import ModelConfig

15 from vllm.multimodal import MULTIMODAL_REGISTRY

17 +from ...registry import HF_EXAMPLE_MODELS

18 from ...utils import build_model_context

20 MODEL_ID = "Qwen/Qwen3-VL-4B-Instruct"

@@ -186,6 +188,89 @@ def test_processor_multi_video_list_kwargs(

188 )

189

190

191 +def _build_video_embeds_mm_data(

192 + hidden_size: int,

193 + grid_thw: tuple[int, int, int] = (2, 4, 4),

194 +) -> dict[str, Any]:

195 + """Create an embeds-only video item as an EPD consumer receives it:

196 + pre-computed embeddings plus the metadata published by the encoder."""

197 + import torch

198 +

199 + t, h, w = grid_thw

200 + num_tokens = t * h * w // 4 # spatial_merge_size ** 2

201 + return {

202 + "video": {

203 + "video_embeds": torch.zeros(num_tokens, hidden_size),

204 + "video_grid_thw": torch.tensor([grid_thw]),

205 + "timestamps": torch.tensor([[float(i) for i in range(t)]]),

206 + }

207 + }

208 +

209 +

210 +@pytest.mark.parametrize("model_id", [MODEL_ID])

211 +def test_processor_video_embeds_with_timestamps(model_id: str) -> None:

212 + """Embeds-only video input must size the placeholder range from the

213 + grid and the real timestamps published by the encoder (EC consumer

214 + path); synthesized or missing timestamps would change the token count

215 + and break embedding merging downstream."""

216 + # `build_model_context` forces enable_mm_embeds off, so build the

217 + # config directly; keep the same online-availability skip behavior.

218 + HF_EXAMPLE_MODELS.find_hf_info(model_id).check_available_online(on_fail="skip")

219 + model_config = ModelConfig(

220 + model_id,

221 + runner="generate",

222 + limit_mm_per_prompt={"image": 0, "video": 1},

223 + enable_mm_embeds=True,

224 + )

225 + processor = MULTIMODAL_REGISTRY.create_processor(model_config)

226 + tokenizer = processor.info.get_tokenizer()

227 +

228 + grid_thw = (2, 4, 4)

229 + mm_data = _build_video_embeds_mm_data(

230 + model_config.get_inputs_embeds_size(), grid_thw

231 + )

232 +

233 + prompt = "<|vision_start|><|video_pad|><|vision_end|>"

234 + processed = processor(

235 + prompt,

236 + mm_items=processor.info.parse_mm_data(mm_data),

237 + hf_processor_mm_kwargs={},

238 + )

239 +

240 + video_phs = processed["mm_placeholders"].get("video", [])

241 + assert len(video_phs) == 1, (

242 + f"Expected exactly 1 video placeholder, got {len(video_phs)}"

243 + )

244 +

245 + t, h, w = grid_thw

246 + tokens_per_frame = h * w // 4

247 + timestamp_tokens = sum(

248 + len(tokenizer.encode(f"<{float(i):.1f} seconds>", add_special_tokens=False))

249 + for i in range(t)

250 + )

251 + # per frame: timestamp tokens + vision_start + video tokens + vision_end

252 + expected_len = timestamp_tokens + t * (tokens_per_frame + 2)

253 + assert video_phs[0].length == expected_len

254 +

255 +

256 +@pytest.mark.parametrize("model_id", [MODEL_ID])

257 +def test_processor_video_embeds_missing_timestamps(model_id: str) -> None:

258 + """Timestamps are required metadata for video embeds: they size the

259 + placeholder range, so omitting them must fail loudly at parse time

260 + instead of silently producing a wrong prompt."""

261 + ctx = build_model_context(

262 + model_id,

263 + limit_mm_per_prompt={"image": 0, "video": 1},

264 + )

265 + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)

266 +

267 + mm_data = _build_video_embeds_mm_data(ctx.model_config.get_inputs_embeds_size())

268 + del mm_data["video"]["timestamps"]

269 +

270 + with pytest.raises(ValueError, match="timestamps"):

271 + processor.info.parse_mm_data(mm_data)

272 +

273 +

274 @pytest.mark.parametrize("model_id", [MODEL_ID])

275 def test_dummy_video_spreads_budget_when_frame_cap_enabled(model_id: str) -> None:

276 """Regression test for memory profiling with ``cap_pixels_per_frame``.

vllm/distributed/ec_transfer/ec_connector/example_connector.py

+10 -5

@@ -14,6 +14,7 @@ from vllm.distributed.ec_transfer.ec_connector.base import (

14 ECConnectorRole,

15 )

16 from vllm.logger import init_logger

17 +from vllm.utils.collection_utils import is_list_of

18 from vllm.v1.core.sched.output import SchedulerOutput

20 if TYPE_CHECKING:

@@ -221,11 +222,15 @@ class ECExampleConnector(ECConnectorBase):

222 # back to processing the media itself.

223 if feature.data is not None:

224 wanted = self._placeholder_metadata_fields(feature.modality)

224 - metadata = {

225 - key: value.tolist()

226 - for key, value in feature.data.get_data().items()

227 - if key in wanted and isinstance(value, torch.Tensor)

228 - }

225 + for key, value in feature.data.get_data().items():

226 + if key not in wanted:

227 + continue

228 + if isinstance(value, torch.Tensor):

229 + metadata[key] = value.tolist()

230 + elif is_list_of(value, (int, float)):

231 + # Some metadata (e.g. Qwen3-VL video timestamps) is

232 + # produced as a plain list rather than a tensor.

233 + metadata[key] = value

234 items.append({"mm_hash": feature.identifier, **metadata})

235

236 if not items:

vllm/entrypoints/chat_utils.py

+115

@@ -178,6 +178,22 @@ class ChatCompletionContentPartAudioEmbedsParam(TypedDict, total=False):

178 """

179

180

181 +class ChatCompletionContentPartVideoEmbedsParam(TypedDict, total=False):

182 + video_embeds: str | dict[str, str] | None

183 + """

184 + The video embeddings. It can be either:

185 + - A single base64 string representing a serialized torch tensor.

186 + - A dictionary where each value is a base64 string.

187 + """

188 + type: Required[Literal["video_embeds"]]

189 + """The type of the content part."""

190 + uuid: str | None

191 + """

192 + User-provided UUID of a media. User must guarantee that it is properly

193 + generated and unique for different medias.

194 + """

195 +

196 +

197 class ChatCompletionContentPartPromptEmbedsParam(TypedDict, total=False):

198 data: Required[str]

199 """

@@ -324,6 +340,7 @@ ChatCompletionContentPartParam: TypeAlias = (

340 | CustomChatCompletionContentSimpleImageParam

341 | ChatCompletionContentPartImageEmbedsParam

342 | ChatCompletionContentPartAudioEmbedsParam

343 + | ChatCompletionContentPartVideoEmbedsParam

344 | ChatCompletionContentPartPromptEmbedsParam

345 | CustomChatCompletionContentSimpleAudioParam

346 | CustomChatCompletionContentSimpleVideoParam

@@ -415,6 +432,7 @@ ModalityStr = Literal[

432 "video",

433 "image_embeds",

434 "audio_embeds",

435 + "video_embeds",

436 "vision_chunk",

437 "prompt_embeds",

438 ]

@@ -755,6 +773,11 @@ def _resolve_items(

773 "Mixing raw audio and embedding inputs is not allowed",

774 parameter="audio_embeds",

775 )

776 + if "video" in items_by_modality and "video_embeds" in items_by_modality:

777 + raise VLLMValidationError(

778 + "Mixing raw video and embedding inputs is not allowed",

779 + parameter="video_embeds",

780 + )

781 # `prompt_embeds` bypasses HF MM processors. Every other modality requires one.

782 processor_modalities = items_by_modality.keys() - {"prompt_embeds"}

783 if processor_modalities and mm_processor is None:

@@ -789,6 +812,14 @@ def _resolve_items(

812 if "video" in items_by_modality:

813 mm_data["video"] = [data for data, uuid in items_by_modality["video"]]

814 mm_uuids["video"] = [uuid for data, uuid in items_by_modality["video"]]

815 + if "video_embeds" in items_by_modality:

816 + assert mm_processor is not None

817 + mm_data["video"] = _get_embeds_data(

818 + "video",

819 + [data for data, uuid in items_by_modality["video_embeds"]],

820 + mm_processor,

821 + )

822 + mm_uuids["video"] = [uuid for data, uuid in items_by_modality["video_embeds"]]

823 if "vision_chunk" in items_by_modality:

824 assert mm_processor is not None

825 # Process vision_chunk items - extract from (data, modality) tuples

@@ -941,6 +972,14 @@ class BaseMultiModalContentParser(ABC):

972 def parse_video(self, video_url: str | None, uuid: str | None = None) -> None:

973 raise NotImplementedError

974

975 + @abstractmethod

976 + def parse_video_embeds(

977 + self,

978 + video_embeds: str | dict[str, str] | None,

979 + uuid: str | None = None,

980 + ) -> None:

981 + raise NotImplementedError

982 +

983

984 class MultiModalContentParser(BaseMultiModalContentParser):

985 def __init__(

@@ -1096,6 +1135,32 @@ class MultiModalContentParser(BaseMultiModalContentParser):

1135 audio_placeholder = self._tracker.add("audio", (audio, uuid))

1136 self._add_placeholder("audio", audio_placeholder)

1137

1138 + def parse_video_embeds(

1139 + self,

1140 + video_embeds: str | dict[str, str] | None,

1141 + uuid: str | None = None,

1142 + ) -> None:

1143 + mm_config = self.model_config.get_multimodal_config()

1144 + if not mm_config.enable_mm_embeds:

1145 + raise VLLMValidationError(

1146 + "You must set `--enable-mm-embeds` to input `video_embeds`",

1147 + parameter="video_embeds",

1148 + )

1149 +

1150 + if isinstance(video_embeds, dict):

1151 + embeds = {

1152 + k: self._connector.fetch_video_embedding(v)

1153 + for k, v in video_embeds.items()

1154 + }

1155 + placeholder = self._tracker.add("video_embeds", (embeds, uuid))

1156 + elif isinstance(video_embeds, str):

1157 + embedding = self._connector.fetch_video_embedding(video_embeds)

1158 + placeholder = self._tracker.add("video_embeds", (embedding, uuid))

1159 + else:

1160 + placeholder = self._tracker.add("video_embeds", (None, uuid))

1161 +

1162 + self._add_placeholder("video", placeholder)

1163 +

1164

1165 class AsyncMultiModalContentParser(BaseMultiModalContentParser):

1166 def __init__(

@@ -1304,6 +1369,43 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):

1369 )

1370 self._add_placeholder("audio", audio_placeholder)

1371

1372 + def parse_video_embeds(

1373 + self,

1374 + video_embeds: str | dict[str, str] | None,

1375 + uuid: str | None = None,

1376 + ) -> None:

1377 + mm_config = self.model_config.get_multimodal_config()

1378 + if not mm_config.enable_mm_embeds:

1379 + raise VLLMValidationError(

1380 + "You must set `--enable-mm-embeds` to input `video_embeds`",

1381 + parameter="video_embeds",

1382 + )

1383 +

1384 + placeholder = self._tracker.add(

1385 + "video_embeds",

1386 + partial(self._video_embeds_with_uuid_async, video_embeds, uuid),

1387 + )

1388 + self._add_placeholder("video", placeholder)

1389 +

1390 + async def _video_embeds_with_uuid_async(

1391 + self,

1392 + video_embeds: str | dict[str, str] | None,

1393 + uuid: str | None,

1394 + ):

1395 + if isinstance(video_embeds, dict):

1396 + tensors = await asyncio.gather(

1397 + *(

1398 + self._connector.fetch_video_embedding_async(v)

1399 + for v in video_embeds.values()

1400 + )

1401 + )

1402 + embeds = dict(zip(video_embeds, tensors))

1403 + elif isinstance(video_embeds, str):

1404 + embeds = await self._connector.fetch_video_embedding_async(video_embeds)

1405 + else:

1406 + embeds = None

1407 + return embeds, uuid

1408 +

1409

1410 @dataclass

1411 class ChatTemplateConfig:

@@ -1481,6 +1583,7 @@ def _get_full_multimodal_text_prompt(

1583 _TextParser = partial(cast, ChatCompletionContentPartTextParam)

1584 _ImageEmbedsParser = partial(cast, ChatCompletionContentPartImageEmbedsParam)

1585 _AudioEmbedsParser = partial(cast, ChatCompletionContentPartAudioEmbedsParam)

1586 +_VideoEmbedsParser = partial(cast, ChatCompletionContentPartVideoEmbedsParam)

1587 _PromptEmbedsParser = partial(cast, ChatCompletionContentPartPromptEmbedsParam)

1588 _InputAudioParser = partial(cast, ChatCompletionContentPartInputAudioParam)

1589 _RefusalParser = partial(cast, ChatCompletionContentPartRefusalParam)

@@ -1507,6 +1610,7 @@ MM_PARSER_MAP: dict[

1610 "image_url": lambda part: _ImageParser(part).get("image_url", {}).get("url", None),

1611 "image_embeds": lambda part: _ImageEmbedsParser(part).get("image_embeds", None),

1612 "audio_embeds": lambda part: _AudioEmbedsParser(part).get("audio_embeds", None),

1613 + "video_embeds": lambda part: _VideoEmbedsParser(part).get("video_embeds", None),

1614 "prompt_embeds": lambda part: _PromptEmbedsParser(part).get("data", None),

1615 "image_pil": lambda part: _PILImageParser(part).get("image_pil", None),

1616 "audio_url": lambda part: _AudioParser(part).get("audio_url", {}).get("url", None),

@@ -1605,6 +1709,13 @@ def _parse_chat_message_content_mm_part(

1709 )

1710 audio_embeds = audio_params.get("audio_embeds", None)

1711 return "audio_embeds", audio_embeds

1712 + if "video_embeds" in part:

1713 + # "video_embeds" could be None if UUID is provided.

1714 + video_embeds_params = cast( # type: ignore[assignment]

1715 + ChatCompletionContentPartVideoEmbedsParam, part

1716 + )

1717 + video_embeds = video_embeds_params.get("video_embeds", None)

1718 + return "video_embeds", video_embeds

1719 if "prompt_embeds" in part:

1720 prompt_embeds_params = cast( # type: ignore[assignment]

1721 ChatCompletionContentPartPromptEmbedsParam, part

@@ -1780,6 +1891,10 @@ def _parse_chat_message_content_part(

1891 content = cast(str | dict[str, str], content) if content is not None else None

1892 mm_parser.parse_audio_embeds(content, uuid)

1893 modality = "audio"

1894 + elif part_type == "video_embeds":

1895 + content = cast(str | dict[str, str], content) if content is not None else None

1896 + mm_parser.parse_video_embeds(content, uuid)

1897 + modality = "video"

1898 elif part_type == "prompt_embeds":

1899 if not content:

1900 raise VLLMValidationError(

vllm/model_executor/models/qwen3_vl.py

+15 -2

@@ -894,6 +894,19 @@ class Qwen3_VisionTransformer(nn.Module):

894 return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)

895

896

897 +class Qwen3VLMultiModalDataParser(Qwen2VLMultiModalDataParser):

898 + # Timestamps are part of the prompt replacement (each frame group's

899 + # "<X.X seconds>" text), so they size the placeholder range and must

900 + # travel with the grid when embeddings are delivered out of band.

901 + embedding_fields = {

902 + **Qwen2VLMultiModalDataParser.embedding_fields,

903 + "video": {

904 + **Qwen2VLMultiModalDataParser.embedding_fields["video"],

905 + "timestamps": "metadata",

906 + },

907 + }

908 +

909 +

910 class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):

911 def get_hf_config(self):

912 return self.ctx.get_hf_config(Qwen3VLConfig)

@@ -912,7 +925,7 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):

925 return self.get_hf_processor(**kwargs).video_processor

926

927 def get_data_parser(self):

915 - return Qwen2VLMultiModalDataParser(

928 + return Qwen3VLMultiModalDataParser(

929 self.get_hf_config().vision_config.spatial_merge_size,

930 video_needs_metadata=True,

931 expected_hidden_size=self._get_expected_hidden_size(),

@@ -1609,7 +1622,7 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])

1622 def get_video_repl(

1623 *,

1624 tokens_per_frame: list[int],

1612 - timestamps: list[float | int],

1625 + timestamps: list[float | int] | torch.Tensor,

1626 tokenizer: TokenizerLike,

1627 vision_start_token_id: int,

1628 vision_end_token_id: int,

vllm/multimodal/media/connector.py

+26 -1

@@ -35,7 +35,7 @@ from vllm.utils.registry import ExtensionManager

35 from .audio import AudioEmbeddingMediaIO, AudioMediaIO

36 from .base import MediaIO, MediaWithBytes

37 from .image import ImageEmbeddingMediaIO, ImageMediaIO

38 -from .video import VideoMediaIO

38 +from .video import VideoEmbeddingMediaIO, VideoMediaIO

40 logger = init_logger(__name__)

@@ -653,3 +653,28 @@ class MediaConnector:

653 return await loop.run_in_executor(

654 global_thread_pool, audio_embedding_io.load_base64, "", data

655 )

656 +

657 + def fetch_video_embedding(

658 + self,

659 + data: str,

660 + ) -> torch.Tensor:

661 + """

662 + Load video embedding from a URL.

663 + """

664 + video_embedding_io = VideoEmbeddingMediaIO()

665 +

666 + return video_embedding_io.load_base64("", data)

667 +

668 + async def fetch_video_embedding_async(

669 + self,

670 + data: str,

671 + ) -> torch.Tensor:

672 + """

673 + Asynchronously load video embedding from a URL.

674 + """

675 + video_embedding_io = VideoEmbeddingMediaIO()

676 + loop = asyncio.get_running_loop()

677 +

678 + return await loop.run_in_executor(

679 + global_thread_pool, video_embedding_io.load_base64, "", data

680 + )

vllm/multimodal/media/video.py

+47 -1

@@ -1,20 +1,24 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 from functools import partial

4 +from io import BytesIO

5 from pathlib import Path

6 from typing import Any

8 import numpy as np

9 import numpy.typing as npt

10 import pybase64

11 +import torch

12 from PIL import Image

14 from vllm import envs

15 from vllm.logger import init_logger

16 +from vllm.utils.serial_utils import tensor2base64

17 +from vllm.utils.sparse_utils import check_sparse_tensor_invariants_threadsafe

19 from ..video import VIDEO_LOADER_REGISTRY

20 from .base import MediaIO, MediaWithBytes

17 -from .image import ImageMediaIO

21 +from .image import MAGIC_NUMPY_PREFIX, ImageMediaIO

23 logger = init_logger(__name__)

@@ -194,3 +198,45 @@ class VideoMediaIO(MediaIO[MediaWithBytes[tuple[npt.NDArray, dict[str, Any]]]]):

198

199 msg = "Only JPEG format is supported for now."

200 raise NotImplementedError(msg)

201 +

202 +

203 +class VideoEmbeddingMediaIO(MediaIO[torch.Tensor]):

204 + """Video embedding MediaIO implementation.

205 +

206 + Configuration values can be user-provided either by --media-io-kwargs or

207 + by the runtime API field "media_io_kwargs". Ensure proper validation and

208 + error handling.

209 + """

210 +

211 + def __init__(self) -> None:

212 + super().__init__()

213 +

214 + def _load_pickled_torch(self, data: bytes) -> torch.Tensor:

215 + buffer = BytesIO(data)

216 + with check_sparse_tensor_invariants_threadsafe():

217 + tensor = torch.load(buffer, weights_only=True)

218 + return tensor.to_dense()

219 +

220 + def _load_numpy(self, data: bytes) -> torch.Tensor:

221 + with BytesIO(data) as buffer:

222 + return torch.from_numpy(np.load(buffer))

223 +

224 + def load_bytes(self, data: bytes) -> torch.Tensor:

225 + if data[:6] == MAGIC_NUMPY_PREFIX:

226 + return self._load_numpy(data)

227 +

228 + return self._load_pickled_torch(data)

229 +

230 + def load_base64(self, media_type: str, data: str) -> torch.Tensor:

231 + return self.load_bytes(pybase64.b64decode(data, validate=True))

232 +

233 + def load_file(self, filepath: Path) -> torch.Tensor:

234 + if filepath.suffix == ".npy":

235 + return torch.from_numpy(np.load(filepath))

236 +

237 + with check_sparse_tensor_invariants_threadsafe():

238 + tensor = torch.load(filepath, weights_only=True)

239 + return tensor.to_dense()

240 +

241 + def encode_base64(self, media: torch.Tensor) -> str:

242 + return tensor2base64(media)