b1d9337e92 · tduncan/Nvidia-Optimization-Stack-VLLM

tests/v1/ec_connector/integration/run_epd_correctness_test.sh

+3

@@ -183,6 +183,7 @@ run_epd_1e_1pd() {

183 --port "$PREFILL_DECODE_PORT" \

184 --max-model-len "$MAX_MODEL_LEN" \

185 --enforce-eager \

186 + --enable-mm-embeds \

187 --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" \

188 --enable-request-id-headers \

189 --max-num-seqs "$MAX_NUM_SEQS" \

@@ -389,6 +390,7 @@ run_epd_1e_1p_1d() {

390 --port "$PREFILL_PORT" \

391 --max-model-len "$MAX_MODEL_LEN" \

392 --enforce-eager \

393 + --enable-mm-embeds \

394 --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" \

395 --enable-request-id-headers \

396 --max-num-seqs "$MAX_NUM_SEQS" \

@@ -415,6 +417,7 @@ run_epd_1e_1p_1d() {

417 --port "$DECODE_PORT" \

418 --max-model-len "$MAX_MODEL_LEN" \

419 --enforce-eager \

420 + --enable-mm-embeds \

421 --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" \

422 --enable-request-id-headers \

423 --max-num-seqs "$MAX_NUM_SEQS" \

vllm/config/multimodal.py

+6 -6

@@ -236,17 +236,17 @@ class MultiModalConfig:

236 - "direct_rpc": Use msgspec serialization via RPC

237 - "torch_shm": Use torch.multiprocessing shared memory for zero-copy IPC

238 Defaults to "direct_rpc". """

239 - mm_embeds_from_ec_connector: bool = False

239 + allow_missing_mm_embeddings: bool = False

240 """Whether a pre-computed-embedding input may omit the `*_embeds` tensor.

241

242 In an encode/prefill/decode (EPD) deployment the encoder instance publishes

243 - embeddings through the EC connector, so the request that reaches the

244 - prefill/decode instance only needs to carry the grid/size metadata that

245 - sizes the placeholder range — the embeddings themselves come from the

246 - connector, keyed by `mm_hash`.

243 + embeddings through the EC connector. An EC consumer loads those embeddings

244 + from the connector, while a KV consumer receives the resulting prompt KV

245 + cache. Their requests only need the grid/size metadata that sizes the

246 + placeholder range.

247

248 Derived, not user-settable: `VllmConfig.__post_init__` sets this to True

249 - exactly on EC consumers. Everywhere else it stays False so that a request

249 + on EC and KV consumers. Everywhere else it stays False so that a request

250 which forgets its embeddings still fails fast in the frontend, with a clear

251 error, rather than deep inside the model."""

252

vllm/config/vllm.py

+13 -13

@@ -1577,7 +1577,7 @@ class VllmConfig:

1577 )

1578 current_platform.check_and_update_config(self)

1579

1580 - self._resolve_mm_embeds_from_ec_connector()

1580 + self._resolve_allow_missing_mm_embeddings()

1581 self._resolve_mm_processor_device()

1582 self._validate_mm_processor_device()

1583

@@ -2254,14 +2254,13 @@ class VllmConfig:

2254 f"kernel_config={self.kernel_config!r}"

2255 )

2256

2257 - def _resolve_mm_embeds_from_ec_connector(self) -> None:

2258 - """Allow `*_embeds` to be omitted only where the connector supplies them.

2257 + def _resolve_allow_missing_mm_embeddings(self) -> None:

2258 + """Allow `*_embeds` tensors to be omitted on disaggregated consumers.

2259

2260 - That is exactly an EC consumer: the encoder instance publishes the

2261 - embeddings through the EC connector, so the request only has to carry

2262 - the grid metadata that sizes the placeholder range. On every other

2263 - deployment a missing `*_embeds` is a client error and must keep failing

2264 - fast in the frontend.

2260 + An EC consumer loads embeddings from its connector. A KV consumer

2261 + receives the prompt KV produced from those embeddings, so it does not

2262 + need the tensors either. On every other deployment a missing tensor is

2263 + a client error and must keep failing fast in the frontend.

2264 """

2265 model_config = self.model_config

2266 if model_config is None:

@@ -2271,15 +2270,16 @@ class VllmConfig:

2270 return

2271

2272 ec_config = self.ec_transfer_config

2273 + kv_config = self.kv_transfer_config

2274 # Derived, so overwrite unconditionally rather than honouring a value

2275 # that was set by hand.

2276 - mm_config.mm_embeds_from_ec_connector = (

2276 + mm_config.allow_missing_mm_embeddings = (

2277 ec_config is not None and ec_config.is_ec_consumer

2278 - )

2279 - if mm_config.mm_embeds_from_ec_connector:

2278 + ) or (kv_config is not None and kv_config.is_kv_consumer)

2279 + if mm_config.allow_missing_mm_embeddings:

2280 logger.info_once(

2281 - "EC consumer: pre-computed-embedding inputs may omit the "

2282 - "embedding tensor; embeddings are loaded from the EC connector."

2281 + "EC/KV consumer: pre-computed-embedding inputs may "

2282 + "omit the embedding tensor."

2283 )

2284

2285 def _resolve_mm_processor_device(self) -> None:

vllm/model_executor/models/colqwen3.py

+1 -1

@@ -104,7 +104,7 @@ class ColQwen3ProcessingInfo(Qwen3VLProcessingInfo):

104 spatial_merge_size,

105 video_needs_metadata=self._supports_video,

106 expected_hidden_size=self._get_expected_hidden_size(),

107 - embeds_from_ec_connector=self.embeds_from_ec_connector,

107 + allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,

108 )

109

110

vllm/model_executor/models/colqwen3_5.py

+1 -1

@@ -101,7 +101,7 @@ class ColQwen3_5ProcessingInfo(Qwen3_5ProcessingInfo):

101 spatial_merge_size,

102 video_needs_metadata=self._supports_video,

103 expected_hidden_size=self._get_expected_hidden_size(),

104 - embeds_from_ec_connector=self.embeds_from_ec_connector,

104 + allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,

105 )

106

107

vllm/model_executor/models/hunyuan_vision.py

+1 -1

@@ -599,7 +599,7 @@ class HunYuanVLProcessingInfo(BaseProcessingInfo):

599 def get_data_parser(self):

600 return HunYuanVLMultiModalDataParser(

601 expected_hidden_size=self._get_expected_hidden_size(),

602 - embeds_from_ec_connector=self.embeds_from_ec_connector,

602 + allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,

603 )

604

605 def get_supported_mm_limits(self) -> Mapping[str, int | None]:

vllm/model_executor/models/keye.py

+1 -1

@@ -936,7 +936,7 @@ class KeyeProcessingInfo(BaseProcessingInfo):

936 def get_data_parser(self):

937 return KeyeMultiModalDataParser(

938 expected_hidden_size=self._get_expected_hidden_size(),

939 - embeds_from_ec_connector=self.embeds_from_ec_connector,

939 + allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,

940 )

941

942 def get_supported_mm_limits(

vllm/model_executor/models/keye_vl1_5.py

+1 -1

@@ -363,7 +363,7 @@ class KeyeVL1_5ProcessingInfo(KeyeProcessingInfo):

363 def get_data_parser(self):

364 return KeyeVL1_5MultiModalDataParser(

365 expected_hidden_size=self._get_expected_hidden_size(),

366 - embeds_from_ec_connector=self.embeds_from_ec_connector,

366 + allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,

367 )

368

369 def get_max_frame_per_video(self) -> int:

vllm/model_executor/models/llava_onevision2.py

+1 -1

@@ -1279,7 +1279,7 @@ class LlavaOnevision2ProcessingInfo(BaseProcessingInfo):

1279 return LlavaOnevision2MultiModalDataParser(

1280 self.get_hf_config().vision_config.spatial_merge_size,

1281 video_needs_metadata=True,

1282 - embeds_from_ec_connector=self.embeds_from_ec_connector,

1282 + allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,

1283 )

1284

1285 def get_hf_processor(self, **kwargs: object):

vllm/model_executor/models/minicpmv.py

+1 -1

@@ -653,7 +653,7 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):

653 def get_data_parser(self):

654 return MiniCPMVMultiModalDataParser(

655 expected_hidden_size=self._get_expected_hidden_size(),

656 - embeds_from_ec_connector=self.embeds_from_ec_connector,

656 + allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,

657 )

658

659 def get_model_version(self):

vllm/model_executor/models/opencua.py

+1 -1

@@ -57,7 +57,7 @@ class OpenCUAProcessingInfo(Qwen2VLProcessingInfo):

57 return Qwen2VLMultiModalDataParser(

58 self.get_hf_config().vision_config.spatial_merge_size,

59 expected_hidden_size=self._get_expected_hidden_size(),

60 - embeds_from_ec_connector=self.embeds_from_ec_connector,

60 + allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,

61 )

62

63 def get_hf_config(self):

vllm/model_executor/models/qwen2_5_omni_thinker.py

+1 -1

@@ -373,7 +373,7 @@ class Qwen2_5OmniThinkerProcessingInfo(

373 target_sr=feature_extractor.sampling_rate,

374 target_channels=self.get_target_channels(),

375 expected_hidden_size=self._get_expected_hidden_size(),

376 - embeds_from_ec_connector=self.embeds_from_ec_connector,

376 + allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,

377 )

378

379 def get_target_channels(self) -> int:

vllm/model_executor/models/qwen2_vl.py

+1 -1

@@ -857,7 +857,7 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):

857 return Qwen2VLMultiModalDataParser(

858 self.get_hf_config().vision_config.spatial_merge_size,

859 expected_hidden_size=self._get_expected_hidden_size(),

860 - embeds_from_ec_connector=self.embeds_from_ec_connector,

860 + allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,

861 )

862

863 def get_supported_mm_limits(self) -> Mapping[str, int | None]:

vllm/model_executor/models/qwen3_vl.py

+1 -1

@@ -910,7 +910,7 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):

910 self.get_hf_config().vision_config.spatial_merge_size,

911 video_needs_metadata=True,

912 expected_hidden_size=self._get_expected_hidden_size(),

913 - embeds_from_ec_connector=self.embeds_from_ec_connector,

913 + allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,

914 )

915

916 def _get_vision_info(

vllm/multimodal/parse.py

+6 -7

@@ -554,10 +554,9 @@ class MultiModalDataParser:

554 embedding inputs. If provided, validates that user-supplied

555 embeddings have the correct hidden size to prevent crashes

556 during model inference.

557 - embeds_from_ec_connector (bool): Whether pre-computed embeddings may be

558 - absent from the request because an encode/prefill/decode encoder

559 - instance publishes them through an EC connector instead. Derived by

560 - `BaseProcessingInfo.embeds_from_ec_connector`.

557 + allow_missing_mm_embeddings (bool): Whether pre-computed embedding

558 + tensors may be absent from the request on a disaggregated consumer.

559 + Derived by `BaseProcessingInfo.allow_missing_mm_embeddings`.

560 """

561

562 embedding_fields: Mapping[str, Mapping[str, EmbeddingFieldRole]] = {}

@@ -594,7 +593,7 @@ class MultiModalDataParser:

593 """

594 metadata = self.placeholder_metadata_fields(modality)

595 values = set(self.embedding_fields.get(modality, {})) - metadata

597 - if self.embeds_from_ec_connector:

596 + if self.allow_missing_mm_embeddings:

597 return metadata, values

598 return metadata | values, set()

599

@@ -606,11 +605,11 @@ class MultiModalDataParser:

605 audio_resample_method: Literal["pyav", "scipy", "soxr"] = "pyav",

606 video_needs_metadata: bool = False,

607 expected_hidden_size: int | None = None,

609 - embeds_from_ec_connector: bool = False,

608 + allow_missing_mm_embeddings: bool = False,

609 ) -> None:

610 super().__init__()

611

613 - self.embeds_from_ec_connector = embeds_from_ec_connector

612 + self.allow_missing_mm_embeddings = allow_missing_mm_embeddings

613

614 self.audio_resampler = AudioResampler(

615 target_sr=target_sr,

vllm/multimodal/processing/context.py

+4 -9

@@ -367,15 +367,10 @@ class BaseProcessingInfo:

367 return None

368

369 @property

370 - def embeds_from_ec_connector(self) -> bool:

371 - """Whether pre-computed embeddings may arrive outside the request.

372 -

373 - True only on an EC consumer, where an encode/prefill/decode encoder

374 - instance publishes them through the connector instead, so the request

375 - carries only the metadata that sizes the placeholder range.

376 - """

370 + def allow_missing_mm_embeddings(self) -> bool:

371 + """Whether pre-computed embedding tensors may be omitted."""

372 mm_config = self.ctx.model_config.multimodal_config

378 - return mm_config is not None and mm_config.mm_embeds_from_ec_connector

373 + return mm_config is not None and mm_config.allow_missing_mm_embeddings

374

375 def get_data_parser(self) -> MultiModalDataParser:

376 """

@@ -389,7 +384,7 @@ class BaseProcessingInfo:

384 """

385 return MultiModalDataParser(

386 expected_hidden_size=self._get_expected_hidden_size(),

392 - embeds_from_ec_connector=self.embeds_from_ec_connector,

387 + allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,

388 )

389

390 @cached_property

[EPD] Allow KV consumers to omit MM embeddings (#52697)