bd2659a566 · tduncan/Nvidia-Optimization-Stack-VLLM

vllm/model_executor/models/clip.py

-4

@@ -931,13 +931,11 @@ class CLIPEmbeddingModel(nn.Module, SupportsMultiModal, SupportsQuant):

931 embed_input_ids: Callable[[torch.Tensor], torch.Tensor],

932 *,

933 is_multimodal: torch.Tensor | None,

934 - handle_oov_mm_token: bool,

934 ) -> torch.Tensor:

935 inputs_embeds = super()._embed_text_input_ids(

936 input_ids,

937 embed_input_ids,

938 is_multimodal=is_multimodal,

940 - handle_oov_mm_token=handle_oov_mm_token,

939 )

940

941 # NOTE: inputs_embeds in model runner has size text_config.projection_dim

@@ -966,7 +964,6 @@ class CLIPEmbeddingModel(nn.Module, SupportsMultiModal, SupportsQuant):

964 multimodal_embeddings: MultiModalEmbeddings | None = None,

965 *,

966 is_multimodal: torch.Tensor | None = None,

969 - handle_oov_mm_token: bool = False,

967 ) -> torch.Tensor:

968 self._is_text_input = (

969 multimodal_embeddings is None or len(multimodal_embeddings) == 0

@@ -980,7 +977,6 @@ class CLIPEmbeddingModel(nn.Module, SupportsMultiModal, SupportsQuant):

977 input_ids,

978 multimodal_embeddings=multimodal_embeddings,

979 is_multimodal=is_multimodal,

983 - handle_oov_mm_token=handle_oov_mm_token,

980 )

981

982 def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:

vllm/model_executor/models/eagle2_5_vl.py

-2

@@ -416,7 +416,6 @@ class Eagle2_5_VLForConditionalGeneration(

416 multimodal_embeddings: MultiModalEmbeddings | None = None,

417 *,

418 is_multimodal: torch.Tensor | None = None,

419 - handle_oov_mm_token: bool = False,

419 ) -> torch.Tensor:

420 """Embed input IDs with optional multimodal embeddings."""

421 if multimodal_embeddings is None or is_multimodal is None:

@@ -426,7 +425,6 @@ class Eagle2_5_VLForConditionalGeneration(

425 input_ids,

426 multimodal_embeddings=multimodal_embeddings,

427 is_multimodal=is_multimodal,

429 - handle_oov_mm_token=handle_oov_mm_token,

428 )

429

430 def forward(

vllm/model_executor/models/ernie45_vl.py

-2

@@ -1664,7 +1664,6 @@ class Ernie4_5_VLMoeForConditionalGeneration(

1664 multimodal_embeddings: MultiModalEmbeddings | None = None,

1665 *,

1666 is_multimodal: torch.Tensor | None = None,

1667 - handle_oov_mm_token: bool = False,

1667 ) -> torch.Tensor:

1668 if multimodal_embeddings is not None and len(multimodal_embeddings) > 0:

1669 self._set_visual_token_mask(input_ids)

@@ -1677,7 +1676,6 @@ class Ernie4_5_VLMoeForConditionalGeneration(

1676 input_ids,

1677 multimodal_embeddings=multimodal_embeddings,

1678 is_multimodal=is_multimodal,

1680 - handle_oov_mm_token=handle_oov_mm_token,

1679 )

1680

1681 def forward(

vllm/model_executor/models/funasr.py

-1

@@ -975,7 +975,6 @@ class FunASRForConditionalGeneration(

975 multimodal_embeddings: MultiModalEmbeddings | None = None,

976 *,

977 is_multimodal: torch.Tensor | None = None,

978 - handle_oov_mm_token: bool = False,

978 ) -> torch.Tensor:

979 inputs_embeds = self.model.decoder.embed_input_ids(input_ids)

980

vllm/model_executor/models/gemma3_mm.py

+5 -2

@@ -507,6 +507,11 @@ class Gemma3ForConditionalGeneration(

507 self.quant_config = quant_config

508 self.multimodal_config = multimodal_config

509

510 + self.configure_mm_token_handling(

511 + vocab_size=config.text_config.vocab_size,

512 + mm_token_ids=[config.image_token_index],

513 + )

514 +

515 with self._mark_tower_model(vllm_config, "image"):

516 self.vision_tower = SiglipVisionModel(

517 config.vision_config,

@@ -587,7 +592,6 @@ class Gemma3ForConditionalGeneration(

592 multimodal_embeddings: MultiModalEmbeddings | None = None,

593 *,

594 is_multimodal: torch.Tensor | None = None,

590 - handle_oov_mm_token: bool = True,

595 ) -> torch.Tensor:

596 # Early return for text-only inference (no multimodal data)

597 if multimodal_embeddings is None or is_multimodal is None:

@@ -598,7 +602,6 @@ class Gemma3ForConditionalGeneration(

602 input_ids,

603 multimodal_embeddings=multimodal_embeddings,

604 is_multimodal=is_multimodal,

601 - handle_oov_mm_token=handle_oov_mm_token,

605 )

606

607 def forward(

vllm/model_executor/models/gemma3n_mm.py

-2

@@ -685,7 +685,6 @@ class Gemma3nForConditionalGeneration(

685 multimodal_embeddings: MultiModalEmbeddings | None = None,

686 *,

687 is_multimodal: torch.Tensor | None = None,

688 - handle_oov_mm_token: bool = False,

688 ) -> torch.Tensor:

689 # NOTE (NickLucche) Each pass needs tokens to compute PLE so we cache

690 # them here, as the model forward has only access to the input_embeds.

@@ -710,7 +709,6 @@ class Gemma3nForConditionalGeneration(

709 input_ids,

710 multimodal_embeddings=multimodal_embeddings,

711 is_multimodal=is_multimodal,

713 - handle_oov_mm_token=handle_oov_mm_token,

712 )

713

714 def forward(

vllm/model_executor/models/granite_speech.py

+6 -3

@@ -600,6 +600,12 @@ class GraniteSpeechForConditionalGeneration(

600 self.quant_config = quant_config

601 self.cache_config = cache_config

602

603 + # Check for OOV tokens to see if offsets need to be preserved

604 + self.configure_mm_token_handling(

605 + vocab_size=config.text_config.vocab_size,

606 + mm_token_ids=[config.audio_token_index],

607 + )

608 +

609 with self._mark_language_model(vllm_config):

610 # The language model is typically a Granite LLM

611 self.language_model = init_vllm_registered_model(

@@ -793,8 +799,6 @@ class GraniteSpeechForConditionalGeneration(

799 multimodal_embeddings: MultiModalEmbeddings | None = None,

800 *,

801 is_multimodal: torch.Tensor | None = None,

796 - # Multi-modal token ID may exceed vocab size

797 - handle_oov_mm_token: bool = True,

802 ) -> torch.Tensor:

803 # This is to satisfy the type checker for each overload

804 if multimodal_embeddings is None or is_multimodal is None:

@@ -804,7 +808,6 @@ class GraniteSpeechForConditionalGeneration(

808 input_ids,

809 multimodal_embeddings=multimodal_embeddings,

810 is_multimodal=is_multimodal,

807 - handle_oov_mm_token=handle_oov_mm_token,

811 )

812

813 def forward(

vllm/model_executor/models/interfaces.py

+31 -18

@@ -130,6 +130,13 @@ class SupportsMultiModal(Protocol):

130 Set internally by `_mark_tower_model`.

131 """

132

133 + _has_oov_mm_tokens: bool = False

134 + """

135 + In general, this should be set at init time by invoking

136 + `configure_mm_token_handling` models & passing all potentially

137 + OOV multimodal tokens.

138 + """

139 +

140 @classmethod

141 def get_placeholder_str(cls, modality: str, i: int) -> str | None:

142 """

@@ -149,6 +156,17 @@ class SupportsMultiModal(Protocol):

156 """

157 ...

158

159 + def configure_mm_token_handling(self, vocab_size: int, mm_token_ids: list[int]):

160 + """Check if any multimodal tokens are out of vocabulary. If so, we will

161 + explicitly mask all multimodal tokens out when computing text embeddings,

162 + since the multimodal embeddings will be scattered over the results.

163 + """

164 + self._has_oov_mm_tokens = any(tok_id >= vocab_size for tok_id in mm_token_ids)

165 + logger.info(

166 + "Contains out of vocabulary multimodal tokens? %s",

167 + self._has_oov_mm_tokens,

168 + )

169 +

170 def get_language_model(self) -> VllmModel:

171 """

172 Returns the underlying language model used for text generation.

@@ -324,7 +342,6 @@ class SupportsMultiModal(Protocol):

342 multimodal_embeddings: MultiModalEmbeddings,

343 *,

344 is_multimodal: torch.Tensor,

327 - handle_oov_mm_token: bool = False,

345 ) -> Tensor: ...

346

347 def _embed_text_input_ids(

@@ -333,17 +350,14 @@ class SupportsMultiModal(Protocol):

350 embed_input_ids: Callable[[Tensor], Tensor],

351 *,

352 is_multimodal: Tensor | None,

336 - handle_oov_mm_token: bool,

353 ) -> Tensor:

338 - if handle_oov_mm_token and is_multimodal is not None:

339 - is_text = ~is_multimodal

340 - text_embeds = embed_input_ids(input_ids[is_text])

341 -

342 - return torch.empty(

343 - (input_ids.shape[0], text_embeds.shape[1]),

344 - dtype=text_embeds.dtype,

345 - device=text_embeds.device,

346 - ).masked_scatter_(is_text.unsqueeze_(-1), text_embeds)

354 + if is_multimodal is not None and self._has_oov_mm_tokens:

355 + # Force all input IDs to be in vocab; we do this instead of squeezing

356 + # to ensure that any external configuration requiring offset tracking,

357 + # e.g., LoRA, are applied correctly regardless of whether or not

358 + # we have multimodal tokens.

359 + in_vocab_ids = input_ids.masked_fill(is_multimodal, 0)

360 + return embed_input_ids(in_vocab_ids)

361

362 return embed_input_ids(input_ids)

363

@@ -353,7 +367,6 @@ class SupportsMultiModal(Protocol):

367 multimodal_embeddings: MultiModalEmbeddings | None = None,

368 *,

369 is_multimodal: Tensor | None = None,

356 - handle_oov_mm_token: bool = False,

370 ) -> Tensor:

371 """

372 Apply token embeddings to `input_ids`.

@@ -361,19 +374,19 @@ class SupportsMultiModal(Protocol):

374 If `multimodal_embeddings` is passed, scatter them into

375 `input_ids` according to the mask `is_multimodal`.

376

364 - In case the multi-modal token IDs exceed the vocabulary size of

365 - the language model, you can set `handle_oov_mm_token=False`

366 - to avoid calling the language model's `embed_input_ids` method

367 - on those tokens. Note however that doing so increases memory usage

368 - as an additional buffer is needed to hold the input embeddings.

377 + NOTE: If this model has multimodal tokens that are of vocabulary

378 + (i.e., self._has_oov_mm_tokens=True), the input_ids will be copied

379 + and masked to 0 during the forward pass for the text embeddings.

380 """

381 from .utils import _merge_multimodal_embeddings

382

383 + # Get text embeddings first; multimodal embeddings will clobber

384 + # any invalid contents in the indices of multimodal embeddings

385 + # for the in vocabulary and out of vocabulary case.

386 inputs_embeds = self._embed_text_input_ids(

387 input_ids,

388 self.get_language_model().embed_input_ids,

389 is_multimodal=is_multimodal,

376 - handle_oov_mm_token=handle_oov_mm_token,

390 )

391

392 if multimodal_embeddings is None or len(multimodal_embeddings) == 0:

vllm/model_executor/models/interns1.py

-2

@@ -764,7 +764,6 @@ class InternS1ForConditionalGeneration(

764 multimodal_embeddings: MultiModalEmbeddings | None = None,

765 *,

766 is_multimodal: torch.Tensor | None = None,

767 - handle_oov_mm_token: bool = False,

767 ) -> torch.Tensor:

768 if multimodal_embeddings is not None and len(multimodal_embeddings) > 0:

769 self._set_visual_token_mask(input_ids)

@@ -777,7 +776,6 @@ class InternS1ForConditionalGeneration(

776 input_ids,

777 multimodal_embeddings=multimodal_embeddings,

778 is_multimodal=is_multimodal,

780 - handle_oov_mm_token=handle_oov_mm_token,

779 )

780

781 def forward(

vllm/model_executor/models/internvl.py

-2

@@ -1347,7 +1347,6 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA)

1347 multimodal_embeddings: MultiModalEmbeddings | None = None,

1348 *,

1349 is_multimodal: torch.Tensor | None = None,

1350 - handle_oov_mm_token: bool = False,

1350 ) -> torch.Tensor:

1351 if multimodal_embeddings is not None and len(multimodal_embeddings) > 0:

1352 self._set_visual_token_mask(input_ids)

@@ -1360,7 +1359,6 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA)

1359 input_ids,

1360 multimodal_embeddings=multimodal_embeddings,

1361 is_multimodal=is_multimodal,

1363 - handle_oov_mm_token=handle_oov_mm_token,

1362 )

1363

1364 def forward(

vllm/model_executor/models/llava.py

+5

@@ -544,6 +544,11 @@ class LlavaForConditionalGeneration(

544 self.config = config

545 self.multimodal_config = multimodal_config

546

547 + self.configure_mm_token_handling(

548 + vocab_size=config.text_config.vocab_size,

549 + mm_token_ids=[config.image_token_index],

550 + )

551 +

552 # NOTE: These are special cases for Pixtral-12B in the HF-format

553 # https://huggingface.co/mistral-community/pixtral-12b/blob/main/config.json # noqa

554 if (

vllm/model_executor/models/llava_next.py

+5 -3

@@ -270,6 +270,11 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsP

270 self.config = config

271 self.multimodal_config = multimodal_config

272

273 + self.configure_mm_token_handling(

274 + vocab_size=config.text_config.vocab_size,

275 + mm_token_ids=[config.image_token_index],

276 + )

277 +

278 with self._mark_tower_model(vllm_config, "image"):

279 self.vision_tower = init_vision_tower_for_llava(

280 config,

@@ -497,8 +502,6 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsP

502 multimodal_embeddings: MultiModalEmbeddings | None = None,

503 *,

504 is_multimodal: torch.Tensor | None = None,

500 - # Multi-modal token ID may exceed vocab size

501 - handle_oov_mm_token: bool = True,

505 ) -> torch.Tensor:

506 # This is to satisfy the type checker for each overload

507 if multimodal_embeddings is None or is_multimodal is None:

@@ -508,7 +511,6 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsP

511 input_ids,

512 multimodal_embeddings=multimodal_embeddings,

513 is_multimodal=is_multimodal,

511 - handle_oov_mm_token=handle_oov_mm_token,

514 )

515

516 def forward(

vllm/model_executor/models/molmo2.py

-2

@@ -2711,13 +2711,11 @@ class Molmo2ForConditionalGeneration(

2711 multimodal_embeddings: MultiModalEmbeddings | None = None,

2712 *,

2713 is_multimodal: torch.Tensor | None = None,

2714 - handle_oov_mm_token: bool = False,

2714 ) -> torch.Tensor:

2715 inputs_embeds = self._embed_text_input_ids(

2716 input_ids,

2717 self.get_language_model().embed_input_ids,

2718 is_multimodal=is_multimodal,

2720 - handle_oov_mm_token=handle_oov_mm_token,

2719 )

2720

2721 if multimodal_embeddings is None or len(multimodal_embeddings) == 0:

vllm/model_executor/models/nemotron_vl.py

-2

@@ -628,7 +628,6 @@ class LlamaNemotronVLChatModel(nn.Module, SupportsMultiModal, SupportsPP, Suppor

628 multimodal_embeddings: MultiModalEmbeddings | None = None,

629 *,

630 is_multimodal: torch.Tensor | None = None,

631 - handle_oov_mm_token: bool = False,

631 ) -> torch.Tensor:

632 if multimodal_embeddings is not None and len(multimodal_embeddings) > 0:

633 self._set_visual_token_mask(input_ids)

@@ -641,7 +640,6 @@ class LlamaNemotronVLChatModel(nn.Module, SupportsMultiModal, SupportsPP, Suppor

640 input_ids,

641 multimodal_embeddings=multimodal_embeddings,

642 is_multimodal=is_multimodal,

644 - handle_oov_mm_token=handle_oov_mm_token,

643 )

644

645 def forward(

vllm/model_executor/models/phi3v.py

-2

@@ -663,13 +663,11 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsQuant)

663 multimodal_embeddings: MultiModalEmbeddings | None = None,

664 *,

665 is_multimodal: torch.Tensor | None = None,

666 - handle_oov_mm_token: bool = False,

666 ) -> torch.Tensor:

667 inputs_embeds = self._embed_text_input_ids(

668 input_ids,

669 self.embed_tokens,

670 is_multimodal=is_multimodal,

672 - handle_oov_mm_token=handle_oov_mm_token,

671 )

672

673 if multimodal_embeddings is None or len(multimodal_embeddings) == 0:

vllm/model_executor/models/qwen2_5_omni_thinker.py

+9 -3

@@ -1428,11 +1428,19 @@ class Qwen2_5OmniThinkerForConditionalGeneration(

1428 multimodal_embeddings: MultiModalEmbeddings | None = None,

1429 *,

1430 is_multimodal: torch.Tensor | None = None,

1431 - handle_oov_mm_token: bool = False,

1431 ) -> torch.Tensor:

1432 if multimodal_embeddings is None or is_multimodal is None:

1433 return super().embed_input_ids(input_ids)

1434

1435 + inputs_embeds = self._embed_text_input_ids(

1436 + input_ids,

1437 + self.get_language_model().embed_input_ids,

1438 + is_multimodal=is_multimodal,

1439 + )

1440 +

1441 + if len(multimodal_embeddings) == 0:

1442 + return inputs_embeds

1443 +

1444 # Check for audio-in-video: interleaved video and audio tokens

1445 # in the multimodal region. Only use the interleaved path when

1446 # needed; otherwise fall back to the default parent implementation.

@@ -1450,7 +1458,6 @@ class Qwen2_5OmniThinkerForConditionalGeneration(

1458 input_ids,

1459 self.get_language_model().embed_input_ids,

1460 is_multimodal=is_multimodal,

1453 - handle_oov_mm_token=handle_oov_mm_token,

1461 )

1462 return merge_interleaved_embeddings(

1463 inputs_embeds,

@@ -1467,7 +1474,6 @@ class Qwen2_5OmniThinkerForConditionalGeneration(

1474 input_ids,

1475 multimodal_embeddings=multimodal_embeddings,

1476 is_multimodal=is_multimodal,

1470 - handle_oov_mm_token=handle_oov_mm_token,

1477 )

1478

1479 def forward(

vllm/model_executor/models/qwen3_5.py

-2

@@ -672,13 +672,11 @@ class Qwen3_5ForConditionalGeneration(Qwen3VLForConditionalGeneration, IsHybrid)

672 multimodal_embeddings: MultiModalEmbeddings | None = None,

673 *,

674 is_multimodal: torch.Tensor | None = None,

675 - handle_oov_mm_token: bool = False,

675 ) -> torch.Tensor:

676 inputs_embeds = self._embed_text_input_ids(

677 input_ids,

678 self.language_model.embed_input_ids,

679 is_multimodal=is_multimodal,

681 - handle_oov_mm_token=handle_oov_mm_token,

680 )

681

682 if multimodal_embeddings is None or len(multimodal_embeddings) == 0:

vllm/model_executor/models/qwen3_5_mtp.py

-2

@@ -380,13 +380,11 @@ class Qwen3_5MTP(nn.Module, SupportsMultiModal):

380 multimodal_embeddings: MultiModalEmbeddings | None = None,

381 *,

382 is_multimodal: torch.Tensor | None = None,

383 - handle_oov_mm_token: bool = False,

383 ) -> torch.Tensor:

384 inputs_embeds = self._embed_text_input_ids(

385 input_ids,

386 self.model.embed_input_ids,

387 is_multimodal=is_multimodal,

389 - handle_oov_mm_token=handle_oov_mm_token,

388 )

389

390 if multimodal_embeddings is None or len(multimodal_embeddings) == 0:

vllm/model_executor/models/qwen3_asr.py

-2

@@ -389,13 +389,11 @@ class Qwen3ASRForConditionalGeneration(

389 multimodal_embeddings: MultiModalEmbeddings | None = None,

390 *,

391 is_multimodal: torch.Tensor | None = None,

392 - handle_oov_mm_token: bool = False,

392 ) -> torch.Tensor:

393 inputs_embeds = self._embed_text_input_ids(

394 input_ids,

395 self.language_model.embed_input_ids,

396 is_multimodal=is_multimodal,

398 - handle_oov_mm_token=handle_oov_mm_token,

397 )

398

399 if multimodal_embeddings is None or len(multimodal_embeddings) == 0:

vllm/model_executor/models/qwen3_omni_moe_thinker.py

-3

@@ -1851,13 +1851,11 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(

1851 multimodal_embeddings: MultiModalEmbeddings | None = None,

1852 *,

1853 is_multimodal: torch.Tensor | None = None,

1854 - handle_oov_mm_token: bool = False,

1854 ) -> torch.Tensor:

1855 inputs_embeds = self._embed_text_input_ids(

1856 input_ids,

1857 self.language_model.embed_input_ids,

1858 is_multimodal=is_multimodal,

1860 - handle_oov_mm_token=handle_oov_mm_token,

1859 )

1860

1861 if multimodal_embeddings is None or len(multimodal_embeddings) == 0:

@@ -1962,7 +1960,6 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(

1960 input_ids,

1961 multimodal_embeddings=multimodal_embeddings,

1962 is_multimodal=is_multimodal,

1965 - handle_oov_mm_token=handle_oov_mm_token,

1963 )

1964

1965 def forward(

vllm/model_executor/models/qwen3_vl.py

-2

@@ -2301,13 +2301,11 @@ class Qwen3VLForConditionalGeneration(

2301 multimodal_embeddings: MultiModalEmbeddings | None = None,

2302 *,

2303 is_multimodal: torch.Tensor | None = None,

2304 - handle_oov_mm_token: bool = False,

2304 ) -> torch.Tensor:

2305 inputs_embeds = self._embed_text_input_ids(

2306 input_ids,

2307 self.language_model.embed_input_ids,

2308 is_multimodal=is_multimodal,

2310 - handle_oov_mm_token=handle_oov_mm_token,

2309 )

2310

2311 if multimodal_embeddings is None or len(multimodal_embeddings) == 0:

vllm/model_executor/models/siglip.py

-4

@@ -1184,13 +1184,11 @@ class SiglipEmbeddingModel(nn.Module, SupportsMultiModal, SupportsQuant):

1184 embed_input_ids: Callable[[torch.Tensor], torch.Tensor],

1185 *,

1186 is_multimodal: torch.Tensor | None,

1187 - handle_oov_mm_token: bool,

1187 ) -> torch.Tensor:

1188 inputs_embeds = super()._embed_text_input_ids(

1189 input_ids,

1190 embed_input_ids,

1191 is_multimodal=is_multimodal,

1193 - handle_oov_mm_token=handle_oov_mm_token,

1192 )

1193

1194 # NOTE: inputs_embeds in model runner has size text_config.projection_size

@@ -1219,7 +1217,6 @@ class SiglipEmbeddingModel(nn.Module, SupportsMultiModal, SupportsQuant):

1217 multimodal_embeddings: MultiModalEmbeddings | None = None,

1218 *,

1219 is_multimodal: torch.Tensor | None = None,

1222 - handle_oov_mm_token: bool = False,

1220 ) -> torch.Tensor:

1221 self._is_text_input = (

1222 multimodal_embeddings is None or len(multimodal_embeddings) == 0

@@ -1232,7 +1229,6 @@ class SiglipEmbeddingModel(nn.Module, SupportsMultiModal, SupportsQuant):

1229 input_ids,

1230 multimodal_embeddings=multimodal_embeddings,

1231 is_multimodal=is_multimodal,

1235 - handle_oov_mm_token=handle_oov_mm_token,

1232 )

1233

1234 def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:

vllm/model_executor/models/skyworkr1v.py

-2

@@ -877,7 +877,6 @@ class SkyworkR1VChatModel(nn.Module, SupportsMultiModal, SupportsPP):

877 multimodal_embeddings: MultiModalEmbeddings | None = None,

878 *,

879 is_multimodal: torch.Tensor | None = None,

880 - handle_oov_mm_token: bool = False,

880 ) -> torch.Tensor:

881 if multimodal_embeddings is not None and len(multimodal_embeddings) > 0:

882 self._set_visual_token_mask(input_ids)

@@ -890,7 +889,6 @@ class SkyworkR1VChatModel(nn.Module, SupportsMultiModal, SupportsPP):

889 input_ids,

890 multimodal_embeddings=multimodal_embeddings,

891 is_multimodal=is_multimodal,

893 - handle_oov_mm_token=handle_oov_mm_token,

892 )

893

894 def forward(

vllm/model_executor/models/step3_vl.py

+13 -4

@@ -937,7 +937,6 @@ class Step3VLForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP)

937

938 def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:

939 super().__init__()

940 -

940 config = vllm_config.model_config.hf_config

941 multimodal_config = vllm_config.model_config.multimodal_config

942

@@ -945,6 +944,19 @@ class Step3VLForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP)

944 self.multimodal_config = multimodal_config

945 self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"

946

947 + # NOTE: This behavior is consistent with the previous OOV handling,

948 + # but does not currently handle the start/stop toks around the

949 + # image features (<patch_start> <patch_end> <im_start> <im_end>)

950 + # See: https://huggingface.co/stepfun-ai/step3/blob/main/processing_step3v.py#L323

951 + #

952 + # If this becomes an issue or we refactor to handle this using the

953 + # processor info in the future, it would probably be best to handle

954 + # those too.

955 + self.configure_mm_token_handling(

956 + self.config.text_config.vocab_size,

957 + [self.config.image_token_id],

958 + )

959 +

960 with self._mark_tower_model(vllm_config, "image"):

961 self.vision_model = Step3VisionTransformer(

962 config.vision_config,

@@ -1080,8 +1092,6 @@ class Step3VLForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP)

1092 multimodal_embeddings: MultiModalEmbeddings | None = None,

1093 *,

1094 is_multimodal: torch.Tensor | None = None,

1083 - # Multi-modal token ID may exceed vocab size

1084 - handle_oov_mm_token: bool = True,

1095 ) -> torch.Tensor:

1096 # This is to satisfy the type checker for each overload

1097 if multimodal_embeddings is None or is_multimodal is None:

@@ -1091,7 +1101,6 @@ class Step3VLForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP)

1101 input_ids,

1102 multimodal_embeddings=multimodal_embeddings,

1103 is_multimodal=is_multimodal,

1094 - handle_oov_mm_token=handle_oov_mm_token,

1104 )

1105

1106 def forward(

vllm/model_executor/models/terratorch.py

-1

@@ -265,7 +265,6 @@ class Terratorch(nn.Module, IsAttentionFree, SupportsMultiModal):

265 multimodal_embeddings: MultiModalEmbeddings | None = None,

266 *,

267 is_multimodal: torch.Tensor | None = None,

268 - handle_oov_mm_token: bool = False,

268 ) -> torch.Tensor:

269 # We do not really use any input tokens and therefore no embeddings

270 # to be calculated. However, due to the mandatory token ids in

vllm/model_executor/models/ultravox.py

+5 -3

@@ -551,6 +551,11 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):

551 self.multi_modal_config = multimodal_config

552 assert self.multi_modal_config

553

554 + self.configure_mm_token_handling(

555 + self.config.vocab_size,

556 + [self.config.audio_token_index],

557 + )

558 +

559 self.secondary_weights = []

560 if config.audio_model_id is not None:

561 # this prefix is not for initialization, but for loading weights

@@ -707,8 +712,6 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):

712 multimodal_embeddings: MultiModalEmbeddings | None = None,

713 *,

714 is_multimodal: torch.Tensor | None = None,

710 - # Multi-modal token ID may exceed vocab size

711 - handle_oov_mm_token: bool = True,

715 ) -> torch.Tensor:

716 # This is to satisfy the type checker for each overload

717 if multimodal_embeddings is None or is_multimodal is None:

@@ -718,7 +721,6 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):

721 input_ids,

722 multimodal_embeddings=multimodal_embeddings,

723 is_multimodal=is_multimodal,

721 - handle_oov_mm_token=handle_oov_mm_token,

724 )

725

726 def forward(

vllm/model_executor/models/voxtral_realtime.py

-1

@@ -298,7 +298,6 @@ class VoxtralRealtimeGeneration(VoxtralForConditionalGeneration, SupportsRealtim

298 *,

299 is_multimodal: torch.Tensor | None = None,

300 # Multi-modal token ID may exceed vocab size

301 - handle_oov_mm_token: bool = True,

301 ) -> torch.Tensor:

302 """Pass post-conv embeddings directly as input.

303

vllm/model_executor/models/whisper.py

-1

@@ -996,7 +996,6 @@ class WhisperForConditionalGeneration(

996 multimodal_embeddings: MultiModalEmbeddings | None = None,

997 *,

998 is_multimodal: torch.Tensor | None = None,

999 - handle_oov_mm_token: bool = False,

999 ) -> torch.Tensor:

1000 # This method just returns the decoder sequence embeddings since

1001 # Whisper does not have encoder text tokens.

Increase Flexibility for OOV Multimodal Token Handling (#34858)