| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | # Copyright 2024 The Qwen team. |
| 4 | # Copyright 2023 The vLLM team. |
| 5 | # Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. |
| 6 | # |
| 7 | # This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX |
| 8 | # and OPT implementations in this library. It has been modified from its |
| 9 | # original forms to accommodate minor architectural differences compared |
| 10 | # to GPT-NeoX and OPT used by the Meta AI team that trained the model. |
| 11 | # |
| 12 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 13 | # you may not use this file except in compliance with the License. |
| 14 | # You may obtain a copy of the License at |
| 15 | # |
| 16 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 17 | # |
| 18 | # Unless required by applicable law or agreed to in writing, software |
| 19 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 20 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 21 | # See the License for the specific language governing permissions and |
| 22 | # limitations under the License. |
| 23 | """Inference-only Qwen2.5-Omni model (thinker part).""" |
| 24 | |
| 25 | from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence |
| 26 | from functools import partial |
| 27 | from typing import Annotated, Any, Literal |
| 28 | |
| 29 | import numpy as np |
| 30 | import torch |
| 31 | import torch.nn as nn |
| 32 | from transformers import PreTrainedConfig |
| 33 | from transformers.feature_extraction_utils import BatchFeature |
| 34 | from transformers.models.qwen2_5_omni.configuration_qwen2_5_omni import ( |
| 35 | Qwen2_5OmniConfig, |
| 36 | Qwen2_5OmniThinkerConfig, |
| 37 | ) |
| 38 | from transformers.models.qwen2_5_omni.modeling_qwen2_5_omni import ( |
| 39 | Qwen2_5OmniAudioEncoder, |
| 40 | ) |
| 41 | from transformers.models.qwen2_5_omni.processing_qwen2_5_omni import ( |
| 42 | Qwen2_5OmniProcessor, |
| 43 | ) |
| 44 | from transformers.models.whisper import WhisperFeatureExtractor |
| 45 | from typing_extensions import TypeVar |
| 46 | |
| 47 | from vllm.config import VllmConfig |
| 48 | from vllm.config.multimodal import MultiModalDummyOptions |
| 49 | from vllm.forward_context import set_forward_context |
| 50 | from vllm.inputs import ModalityData, MultiModalDataDict |
| 51 | from vllm.logger import init_logger |
| 52 | from vllm.model_executor.models.module_mapping import MultiModelKeys |
| 53 | from vllm.model_executor.models.qwen2_5_vl import ( |
| 54 | Qwen2_5_VisionTransformer, |
| 55 | Qwen2_5_VLImageEmbeddingInputs, |
| 56 | Qwen2_5_VLImageInputs, |
| 57 | Qwen2_5_VLImagePixelInputs, |
| 58 | Qwen2_5_VLProcessingInfo, |
| 59 | Qwen2_5_VLVideoEmbeddingInputs, |
| 60 | Qwen2_5_VLVideoInputs, |
| 61 | Qwen2_5_VLVideoPixelInputs, |
| 62 | ) |
| 63 | from vllm.model_executor.models.qwen2_audio import ( |
| 64 | Qwen2AudioProcessingInfo, |
| 65 | _get_feat_extract_output_lengths, |
| 66 | ) |
| 67 | from vllm.model_executor.models.qwen2_vl import ( |
| 68 | Qwen2VLMultiModalDataParser, |
| 69 | Qwen2VLProcessingInfo, |
| 70 | ) |
| 71 | from vllm.multimodal import MULTIMODAL_REGISTRY |
| 72 | from vllm.multimodal.inputs import ( |
| 73 | ImageItem, |
| 74 | MultiModalFeatureSpec, |
| 75 | MultiModalFieldConfig, |
| 76 | MultiModalKwargsItems, |
| 77 | ) |
| 78 | from vllm.multimodal.parse import ( |
| 79 | AudioProcessorItems, |
| 80 | DictEmbeddingItems, |
| 81 | ModalityDataItems, |
| 82 | MultiModalDataItems, |
| 83 | ) |
| 84 | from vllm.multimodal.processing import ( |
| 85 | BaseDummyInputsBuilder, |
| 86 | BaseProcessingInfo, |
| 87 | ) |
| 88 | from vllm.multimodal.processing.processor import ( |
| 89 | BaseMultiModalProcessor, |
| 90 | HFMultiModalInputs, |
| 91 | MultiModalProcessingResult, |
| 92 | MultiModalPromptUpdates, |
| 93 | PlaceholderFeaturesInfo, |
| 94 | PromptReplacement, |
| 95 | PromptUpdate, |
| 96 | PromptUpdateDetails, |
| 97 | ) |
| 98 | from vllm.sequence import IntermediateTensors |
| 99 | from vllm.utils.gpu_sync_debug import gpu_sync_allowed |
| 100 | from vllm.utils.tensor_schema import TensorSchema, TensorShape |
| 101 | from vllm.utils.torch_utils import async_tensor_h2d |
| 102 | |
| 103 | from .interfaces import ( |
| 104 | MultiModalEmbeddings, |
| 105 | SupportsLoRA, |
| 106 | SupportsMRoPE, |
| 107 | SupportsMultiModal, |
| 108 | SupportsPP, |
| 109 | ) |
| 110 | from .utils import ( |
| 111 | AutoWeightsLoader, |
| 112 | WeightsMapper, |
| 113 | init_vllm_registered_model, |
| 114 | maybe_prefix, |
| 115 | split_list_into_ranges, |
| 116 | ) |
| 117 | |
| 118 | try: |
| 119 | import flash_attn |
| 120 | except (ImportError, ModuleNotFoundError): |
| 121 | flash_attn = None |
| 122 | |
| 123 | logger = init_logger(__name__) |
| 124 | |
| 125 | |
| 126 | def _get_feature_tensor(feature: MultiModalFeatureSpec, key: str) -> torch.Tensor: |
| 127 | data = feature.data |
| 128 | assert data is not None |
| 129 | tensor = data[key].data |
| 130 | assert isinstance(tensor, torch.Tensor) |
| 131 | return tensor |
| 132 | |
| 133 | |
| 134 | def unpad_and_flat_audio_features( |
| 135 | input_audio_features: torch.Tensor, |
| 136 | audio_feature_lengths: torch.Tensor, |
| 137 | ) -> torch.Tensor: |
| 138 | """Unpad and flatten batched audio features.""" |
| 139 | if len(audio_feature_lengths) != input_audio_features.size(0): |
| 140 | raise ValueError( |
| 141 | "Length of audio_feature_lengths must match " |
| 142 | "the batch size of input_audio_features." |
| 143 | ) |
| 144 | unpadded_features = [ |
| 145 | audio_feature[..., :length] |
| 146 | for audio_feature, length in zip(input_audio_features, audio_feature_lengths) |
| 147 | ] |
| 148 | return torch.cat(unpadded_features, dim=-1) |
| 149 | |
| 150 | |
| 151 | def check_interleaved_audio_video( |
| 152 | is_video: torch.Tensor, |
| 153 | is_audio: torch.Tensor, |
| 154 | num_video: int, |
| 155 | num_audio: int, |
| 156 | ) -> bool: |
| 157 | """Check if video and audio positions are interleaved in any per-video span. |
| 158 | |
| 159 | For use_audio_in_video=True, each video placeholder is expanded into one |
| 160 | local span containing only video/audio pad tokens, bounded by non-pad |
| 161 | tokens such as audio_start/audio_end. Check each contiguous V/A span |
| 162 | independently instead of requiring all V/A tokens in the whole sequence to |
| 163 | form one global dense range; otherwise multi-video requests can be |
| 164 | misclassified as non-interleaved because of the boundary tokens between |
| 165 | videos. |
| 166 | """ |
| 167 | if num_video == 0 or num_audio == 0: |
| 168 | return False |
| 169 | |
| 170 | va_pos = (is_video | is_audio).nonzero(as_tuple=True)[0] |
| 171 | if len(va_pos) == 0: |
| 172 | return False |
| 173 | |
| 174 | span_start = 0 |
| 175 | for span_end in range(1, len(va_pos) + 1): |
| 176 | is_last = span_end == len(va_pos) |
| 177 | if not is_last and va_pos[span_end].item() == va_pos[span_end - 1].item() + 1: |
| 178 | continue |
| 179 | |
| 180 | span = va_pos[span_start:span_end] |
| 181 | span_is_video = is_video[span] |
| 182 | span_is_audio = is_audio[span] |
| 183 | if span_is_video.any() and span_is_audio.any(): |
| 184 | video_offsets = span_is_video.nonzero(as_tuple=True)[0] |
| 185 | audio_offsets = span_is_audio.nonzero(as_tuple=True)[0] |
| 186 | if ( |
| 187 | video_offsets[0].item() < audio_offsets[-1].item() |
| 188 | and audio_offsets[0].item() < video_offsets[-1].item() |
| 189 | ): |
| 190 | return True |
| 191 | |
| 192 | span_start = span_end |
| 193 | |
| 194 | return False |
| 195 | |
| 196 | |
| 197 | def merge_interleaved_embeddings( |
| 198 | inputs_embeds: torch.Tensor, |
| 199 | multimodal_embeddings: "MultiModalEmbeddings", |
| 200 | is_video: torch.Tensor, |
| 201 | is_audio: torch.Tensor, |
| 202 | is_multimodal: torch.Tensor, |
| 203 | ) -> torch.Tensor: |
| 204 | """Merge embeddings for interleaved audio-in-video sequences. |
| 205 | |
| 206 | When use_audio_in_video=True, video and audio tokens are interleaved in |
| 207 | the token sequence, but embeddings are provided as separate contiguous |
| 208 | tensors. This function scatters each modality by the ``modality`` attribute |
| 209 | attached to each embedding tensor (set during encoder gather) and also |
| 210 | supports image embeddings in the same interleaved request. |
| 211 | |
| 212 | Args: |
| 213 | inputs_embeds: The input embeddings tensor to merge into. |
| 214 | multimodal_embeddings: List of embedding tensors (video, audio, image). |
| 215 | is_video: Boolean mask for video token positions. |
| 216 | is_audio: Boolean mask for audio token positions. |
| 217 | is_multimodal: Boolean mask for all multimodal token positions. |
| 218 | |
| 219 | Returns: |
| 220 | The merged inputs_embeds tensor with multimodal embeddings scattered |
| 221 | to their correct positions. |
| 222 | |
| 223 | """ |
| 224 | from vllm.multimodal.utils import get_mm_embedding_modalities |
| 225 | |
| 226 | def _merge_embedding_group( |
| 227 | mask: torch.Tensor, |
| 228 | embeddings: Sequence[torch.Tensor], |
| 229 | modality: str, |
| 230 | ) -> None: |
| 231 | num_expected_tokens = mask.sum().item() |
| 232 | num_actual_tokens = sum(embedding.shape[0] for embedding in embeddings) |
| 233 | if num_actual_tokens != num_expected_tokens: |
| 234 | raise ValueError( |
| 235 | f"Attempted to assign {num_actual_tokens} {modality} tokens " |
| 236 | f"to {num_expected_tokens} placeholders" |
| 237 | ) |
| 238 | if embeddings: |
| 239 | inputs_embeds[mask] = torch.cat(list(embeddings), dim=0) |
| 240 | |
| 241 | # Qwen Omni multimodal placeholders are image/video/audio; after excluding |
| 242 | # video and audio positions, the remaining multimodal positions are images. |
| 243 | is_image = is_multimodal & ~is_video & ~is_audio |
| 244 | |
| 245 | embedding_modalities = get_mm_embedding_modalities(multimodal_embeddings) |
| 246 | |
| 247 | video_embeds = [ |
| 248 | embedding |
| 249 | for embedding, modality in zip(multimodal_embeddings, embedding_modalities) |
| 250 | if modality == "video" |
| 251 | ] |
| 252 | audio_embeds = [ |
| 253 | embedding |
| 254 | for embedding, modality in zip(multimodal_embeddings, embedding_modalities) |
| 255 | if modality == "audio" |
| 256 | ] |
| 257 | image_embeds = [ |
| 258 | embedding |
| 259 | for embedding, modality in zip(multimodal_embeddings, embedding_modalities) |
| 260 | if modality == "image" |
| 261 | ] |
| 262 | |
| 263 | _merge_embedding_group(is_video, video_embeds, "video") |
| 264 | _merge_embedding_group(is_audio, audio_embeds, "audio") |
| 265 | _merge_embedding_group(is_image, image_embeds, "image") |
| 266 | |
| 267 | return inputs_embeds |
| 268 | |
| 269 | |
| 270 | class Qwen2_5OmniAudioFeatureInputs(TensorSchema): |
| 271 | """Dimensions: |
| 272 | - na: Number of audios |
| 273 | - nmb: Number of mel bins |
| 274 | - msl: Maximum sequence length |
| 275 | - tsl: Total sequence length |
| 276 | """ |
| 277 | |
| 278 | type: Literal["audio_features"] |
| 279 | input_features: Annotated[ |
| 280 | torch.Tensor | list[torch.Tensor], |
| 281 | TensorShape("nmb", "tsl", dynamic_dims={"tsl"}), |
| 282 | ] |
| 283 | |
| 284 | audio_feature_lengths: Annotated[torch.Tensor, TensorShape("na")] |
| 285 | |
| 286 | feature_attention_mask: Annotated[ |
| 287 | torch.Tensor | list[torch.Tensor], |
| 288 | TensorShape("na", "msl", dynamic_dims={"msl"}), |
| 289 | ] |
| 290 | |
| 291 | |
| 292 | def create_qwen2_5_omni_thinker_field_factory( |
| 293 | spatial_merge_size: int, |
| 294 | ) -> Callable[[Mapping[str, torch.Tensor]], Mapping[str, MultiModalFieldConfig]]: |
| 295 | def _qwen2_5_omni_thinker_field_config(hf_inputs: Mapping[str, torch.Tensor]): |
| 296 | audio_feature_lengths = hf_inputs.get( |
| 297 | "audio_feature_lengths", torch.empty((0,)) |
| 298 | ).flatten() |
| 299 | |
| 300 | image_grid_thw = hf_inputs.get("image_grid_thw", torch.empty((0, 3))) |
| 301 | image_pixel_grid_sizes = image_grid_thw.prod(-1) |
| 302 | image_embed_grid_sizes = ( |
| 303 | image_pixel_grid_sizes // spatial_merge_size // spatial_merge_size |
| 304 | ) |
| 305 | |
| 306 | video_grid_thw = hf_inputs.get("video_grid_thw", torch.empty((0, 3))) |
| 307 | video_grid_sizes = video_grid_thw.prod(-1) |
| 308 | video_embed_grid_sizes = ( |
| 309 | video_grid_sizes // spatial_merge_size // spatial_merge_size |
| 310 | ) |
| 311 | |
| 312 | num_videos = len(video_grid_sizes) |
| 313 | |
| 314 | return dict( |
| 315 | input_audio_features=MultiModalFieldConfig.flat_from_sizes( |
| 316 | "audio", audio_feature_lengths, dim=1 |
| 317 | ), |
| 318 | feature_attention_mask=MultiModalFieldConfig.batched( |
| 319 | "audio", keep_on_cpu=True |
| 320 | ), |
| 321 | audio_feature_lengths=MultiModalFieldConfig.batched( |
| 322 | "audio", keep_on_cpu=True |
| 323 | ), |
| 324 | pixel_values=MultiModalFieldConfig.flat_from_sizes( |
| 325 | "image", image_pixel_grid_sizes |
| 326 | ), |
| 327 | image_embeds=MultiModalFieldConfig.flat_from_sizes( |
| 328 | "image", image_embed_grid_sizes |
| 329 | ), |
| 330 | image_grid_thw=MultiModalFieldConfig.batched("image", keep_on_cpu=True), |
| 331 | pixel_values_videos=MultiModalFieldConfig.flat_from_sizes( |
| 332 | "video", video_grid_sizes |
| 333 | ), |
| 334 | video_embeds=MultiModalFieldConfig.flat_from_sizes( |
| 335 | "video", video_embed_grid_sizes |
| 336 | ), |
| 337 | video_grid_thw=MultiModalFieldConfig.batched("video", keep_on_cpu=True), |
| 338 | second_per_grid_ts=MultiModalFieldConfig.batched("video", keep_on_cpu=True), |
| 339 | use_audio_in_video=MultiModalFieldConfig.shared( |
| 340 | "video", num_videos, keep_on_cpu=True |
| 341 | ), |
| 342 | ) |
| 343 | |
| 344 | return _qwen2_5_omni_thinker_field_config |
| 345 | |
| 346 | |
| 347 | class Qwen2_5OmniThinkerMultiModalDataParser(Qwen2VLMultiModalDataParser): |
| 348 | embedding_fields = { |
| 349 | **Qwen2VLMultiModalDataParser.embedding_fields, |
| 350 | "audio": { |
| 351 | "input_audio_features": "values", |
| 352 | "audio_feature_lengths": "metadata", |
| 353 | }, |
| 354 | } |
| 355 | |
| 356 | def __init__(self, spatial_merge_size: int, *args, **kwargs): |
| 357 | self._spatial_merge_size = spatial_merge_size |
| 358 | super().__init__(self._spatial_merge_size, *args, **kwargs) |
| 359 | |
| 360 | def _parse_audio_data( |
| 361 | self, |
| 362 | data: dict[str, torch.Tensor] | ModalityData[ImageItem], |
| 363 | ) -> ModalityDataItems[Any, Any] | None: |
| 364 | if isinstance(data, dict): |
| 365 | required, optional = self.embedding_field_sets("audio") |
| 366 | data = dict(data) |
| 367 | if "audio_feature_lengths" in data: |
| 368 | data["audio_feature_lengths"] = data["audio_feature_lengths"].flatten() |
| 369 | return DictEmbeddingItems( |
| 370 | data, |
| 371 | modality="audio", |
| 372 | required_fields=required, |
| 373 | optional_fields=optional, |
| 374 | fields_factory=create_qwen2_5_omni_thinker_field_factory( |
| 375 | self._spatial_merge_size |
| 376 | ), |
| 377 | ) |
| 378 | |
| 379 | return super()._parse_audio_data(data) |
| 380 | |
| 381 | |
| 382 | class Qwen2_5OmniThinkerProcessingInfo( |
| 383 | Qwen2AudioProcessingInfo, Qwen2_5_VLProcessingInfo |
| 384 | ): |
| 385 | def get_hf_config(self): |
| 386 | return self.ctx.get_hf_config(Qwen2_5OmniConfig).thinker_config |
| 387 | |
| 388 | def get_hf_processor(self, **kwargs: object) -> Qwen2_5OmniProcessor: |
| 389 | return self.ctx.get_hf_processor( |
| 390 | Qwen2_5OmniProcessor, |
| 391 | **kwargs, |
| 392 | ) |
| 393 | |
| 394 | def get_feature_extractor(self, **kwargs: object): |
| 395 | hf_processor = self.get_hf_processor(**kwargs) |
| 396 | feature_extractor = hf_processor.feature_extractor # type: ignore |
| 397 | assert isinstance(feature_extractor, WhisperFeatureExtractor) |
| 398 | return feature_extractor |
| 399 | |
| 400 | def get_data_parser(self): |
| 401 | feature_extractor = self.get_feature_extractor() |
| 402 | |
| 403 | return Qwen2_5OmniThinkerMultiModalDataParser( |
| 404 | spatial_merge_size=self.get_hf_config().vision_config.spatial_merge_size, |
| 405 | target_sr=feature_extractor.sampling_rate, |
| 406 | target_channels=self.get_target_channels(), |
| 407 | expected_hidden_size=self._get_expected_hidden_size(), |
| 408 | allow_missing_mm_embeddings=self.allow_missing_mm_embeddings, |
| 409 | ) |
| 410 | |
| 411 | def get_target_channels(self) -> int: |
| 412 | """Return target audio channels for Qwen2.5 Omni models (mono).""" |
| 413 | return 1 |
| 414 | |
| 415 | def get_supported_mm_limits(self) -> Mapping[str, int | None]: |
| 416 | return {"audio": None, "image": None, "video": None} |
| 417 | |
| 418 | def get_mm_max_tokens_per_item( |
| 419 | self, |
| 420 | seq_len: int, |
| 421 | mm_counts: Mapping[str, int] | None = None, |
| 422 | ) -> Mapping[str, int]: |
| 423 | mm_counts = mm_counts or {} |
| 424 | requested_modalities = {m for m, c in mm_counts.items() if c > 0} |
| 425 | mm_max_tokens: dict[str, int] = {} |
| 426 | |
| 427 | if requested_modalities & {"image", "video"}: |
| 428 | vl_tokens = Qwen2_5_VLProcessingInfo.get_mm_max_tokens_per_item( |
| 429 | self, |
| 430 | seq_len=seq_len, |
| 431 | mm_counts=mm_counts, |
| 432 | ) |
| 433 | mm_max_tokens.update( |
| 434 | { |
| 435 | m: vl_tokens[m] |
| 436 | for m in ["image", "video"] |
| 437 | if m in requested_modalities |
| 438 | } |
| 439 | ) |
| 440 | |
| 441 | if "audio" in requested_modalities: |
| 442 | audio_tokens = Qwen2AudioProcessingInfo.get_mm_max_tokens_per_item( |
| 443 | self, |
| 444 | seq_len=seq_len, |
| 445 | mm_counts=mm_counts, |
| 446 | ) |
| 447 | mm_max_tokens["audio"] = audio_tokens["audio"] |
| 448 | |
| 449 | return mm_max_tokens |
| 450 | |
| 451 | |
| 452 | _I = TypeVar("_I", bound=BaseProcessingInfo, default=Qwen2_5OmniThinkerProcessingInfo) |
| 453 | _VI = TypeVar( |
| 454 | "_VI", bound=Qwen2VLProcessingInfo, default=Qwen2_5OmniThinkerProcessingInfo |
| 455 | ) |
| 456 | |
| 457 | |
| 458 | class Qwen2_5OmniThinkerDummyInputsBuilder(BaseDummyInputsBuilder[_VI]): |
| 459 | def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str: |
| 460 | num_audios = mm_counts.get("audio", 0) |
| 461 | num_images = mm_counts.get("image", 0) |
| 462 | num_videos = mm_counts.get("video", 0) |
| 463 | |
| 464 | hf_processor = self.info.get_hf_processor() |
| 465 | |
| 466 | audio_token: str = hf_processor.audio_token |
| 467 | image_token: str = hf_processor.image_token |
| 468 | video_token: str = hf_processor.video_token |
| 469 | |
| 470 | return ( |
| 471 | audio_token * num_audios |
| 472 | + image_token * num_images |
| 473 | + video_token * num_videos |
| 474 | ) |
| 475 | |
| 476 | def get_dummy_mm_data( |
| 477 | self, |
| 478 | seq_len: int, |
| 479 | mm_counts: Mapping[str, int], |
| 480 | mm_options: MultiModalDummyOptions, |
| 481 | ) -> MultiModalDataDict: |
| 482 | feature_extractor = self.info.get_hf_processor().feature_extractor |
| 483 | |
| 484 | target_audio_length = ( |
| 485 | min( |
| 486 | feature_extractor.chunk_length, |
| 487 | 30, |
| 488 | ) |
| 489 | * feature_extractor.sampling_rate |
| 490 | ) |
| 491 | |
| 492 | target_width, target_height = self.info.get_image_size_with_most_features() |
| 493 | target_num_frames = self.info.get_num_frames_with_most_features( |
| 494 | seq_len, mm_counts |
| 495 | ) |
| 496 | |
| 497 | mm_data = { |
| 498 | "audio": self._get_dummy_audios( |
| 499 | length=target_audio_length, |
| 500 | num_audios=mm_counts.get("audio", 0), |
| 501 | overrides=mm_options.get("audio"), |
| 502 | ), |
| 503 | "image": self._get_dummy_images( |
| 504 | width=target_width, |
| 505 | height=target_height, |
| 506 | num_images=mm_counts.get("image", 0), |
| 507 | overrides=mm_options.get("image"), |
| 508 | ), |
| 509 | "video": self._get_dummy_videos( |
| 510 | width=target_width, |
| 511 | height=target_height, |
| 512 | num_frames=target_num_frames, |
| 513 | num_videos=mm_counts.get("video", 0), |
| 514 | overrides=mm_options.get("video"), |
| 515 | ), |
| 516 | } |
| 517 | |
| 518 | return mm_data |
| 519 | |
| 520 | |
| 521 | class Qwen2_5OmniThinkerMultiModalProcessor(BaseMultiModalProcessor[_I]): |
| 522 | def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: |
| 523 | return self.dummy_inputs.get_dummy_text(mm_counts) |
| 524 | |
| 525 | def _get_hf_mm_inputs( |
| 526 | self, |
| 527 | mm_items: MultiModalDataItems, |
| 528 | hf_kwargs: Mapping[str, object], |
| 529 | ) -> HFMultiModalInputs: |
| 530 | hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs) |
| 531 | |
| 532 | mm_counts = mm_items.get_all_counts() |
| 533 | if hf_kwargs.get("use_audio_in_video"): |
| 534 | num_videos = mm_counts.get("video", 0) |
| 535 | if mm_counts.get("audio", 0) < num_videos: |
| 536 | raise ValueError( |
| 537 | "Video doesn't have audio track with `audio_in_video=True`" |
| 538 | ) |
| 539 | |
| 540 | if num_videos and "text" in hf_inputs.hf_data: |
| 541 | standalone_mm_counts = dict(mm_counts) |
| 542 | standalone_mm_counts["audio"] -= num_videos |
| 543 | hf_inputs.hf_data["text"] = self._get_hf_mm_text(standalone_mm_counts) |
| 544 | |
| 545 | hf_data = hf_inputs.hf_data |
| 546 | normalized_kwargs = hf_inputs.hf_kwargs |
| 547 | audios = hf_data.pop("audio", []) |
| 548 | |
| 549 | # NOTE: WhisperFeatureExtractor cannot handle empty list of audios |
| 550 | if audios: |
| 551 | hf_data["audio"] = audios |
| 552 | elif ( |
| 553 | normalized_kwargs.get("use_audio_in_video") |
| 554 | and "audio" not in hf_data |
| 555 | and hf_data.get("videos") |
| 556 | ): |
| 557 | raise ValueError( |
| 558 | "Video doesn't have audio track with `audio_in_video=True`" |
| 559 | ) |
| 560 | |
| 561 | merged = self.info.ctx.get_merged_mm_kwargs(normalized_kwargs) |
| 562 | if hf_data.get("videos") and ( |
| 563 | merged.keys() & {"size", "min_pixels", "max_pixels"} |
| 564 | ): |
| 565 | normalized_kwargs = dict(normalized_kwargs) |
| 566 | video_size = dict(self.info.get_hf_processor().video_processor.size) |
| 567 | size_override = merged.get("size") |
| 568 | if size_override is not None: |
| 569 | video_size = video_size | size_override |
| 570 | min_pixels = merged.get("min_pixels") |
| 571 | if min_pixels is not None: |
| 572 | video_size["shortest_edge"] = min_pixels |
| 573 | max_pixels = merged.get("max_pixels") |
| 574 | if max_pixels is not None: |
| 575 | video_size["longest_edge"] = max_pixels |
| 576 | normalized_kwargs["size"] = video_size |
| 577 | |
| 578 | return hf_inputs._replace(hf_kwargs=normalized_kwargs) |
| 579 | |
| 580 | def _get_mm_fields_config( |
| 581 | self, |
| 582 | hf_inputs: BatchFeature, |
| 583 | hf_processor_mm_kwargs: Mapping[str, object], |
| 584 | ) -> Mapping[str, MultiModalFieldConfig]: |
| 585 | return create_qwen2_5_omni_thinker_field_factory( |
| 586 | self.info.get_hf_config().vision_config.spatial_merge_size |
| 587 | )(hf_inputs) |
| 588 | |
| 589 | def _derive_audio_from_video_placeholders( |
| 590 | self, |
| 591 | placeholders: Mapping[str, list[PlaceholderFeaturesInfo]], |
| 592 | mm_prompt_updates: MultiModalPromptUpdates, |
| 593 | ) -> Mapping[str, list[PlaceholderFeaturesInfo]]: |
| 594 | """Helper to derive audio placeholders from video placeholders when |
| 595 | use_audio_in_video=True. |
| 596 | """ |
| 597 | if "video" not in placeholders: |
| 598 | return placeholders |
| 599 | |
| 600 | # Validate audio and video counts match |
| 601 | num_videos = len(placeholders["video"]) |
| 602 | num_audios = len(mm_prompt_updates.get("audio", [])) |
| 603 | if num_audios != num_videos: |
| 604 | raise ValueError( |
| 605 | f"use_audio_in_video requires equal number of audio and video " |
| 606 | f"items, got {num_audios=}, {num_videos=}" |
| 607 | ) |
| 608 | |
| 609 | tokenizer = self.info.get_tokenizer() |
| 610 | processor = self.info.get_hf_processor() |
| 611 | audio_token_id = tokenizer.get_vocab()[processor.audio_token] |
| 612 | video_token_id = tokenizer.get_vocab()[processor.video_token] |
| 613 | |
| 614 | result_placeholders = dict(placeholders) |
| 615 | audio_placeholders = [] |
| 616 | video_placeholders = [] |
| 617 | |
| 618 | # Each video is paired with one audio |
| 619 | for video_idx, video_placeholder in enumerate(placeholders["video"]): |
| 620 | # Create is_embed mask selecting only audio tokens |
| 621 | audio_is_embed = torch.tensor(video_placeholder.tokens) == audio_token_id |
| 622 | |
| 623 | # Create is_embed mask selecting only video tokens |
| 624 | video_is_embed = torch.tensor(video_placeholder.tokens) == video_token_id |
| 625 | |
| 626 | audio_placeholder = PlaceholderFeaturesInfo( |
| 627 | modality="audio", |
| 628 | item_idx=video_idx, |
| 629 | start_idx=video_placeholder.start_idx, |
| 630 | tokens=video_placeholder.tokens, |
| 631 | is_embed=audio_is_embed, |
| 632 | ) |
| 633 | audio_placeholders.append(audio_placeholder) |
| 634 | |
| 635 | # Update video placeholder with is_embed mask |
| 636 | video_placeholder_with_mask = PlaceholderFeaturesInfo( |
| 637 | modality="video", |
| 638 | item_idx=video_idx, |
| 639 | start_idx=video_placeholder.start_idx, |
| 640 | tokens=video_placeholder.tokens, |
| 641 | is_embed=video_is_embed, |
| 642 | ) |
| 643 | video_placeholders.append(video_placeholder_with_mask) |
| 644 | |
| 645 | result_placeholders["audio"] = audio_placeholders |
| 646 | result_placeholders["video"] = video_placeholders |
| 647 | return result_placeholders |
| 648 | |
| 649 | def _maybe_apply_prompt_updates( |
| 650 | self, |
| 651 | mm_items: MultiModalDataItems, |
| 652 | mm_res: MultiModalProcessingResult, |
| 653 | ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]: |
| 654 | """Qwen2.5-Omni reimplements this function to handle `use_audio_in_video`.""" |
| 655 | mm_item_counts = mm_items.get_all_counts() |
| 656 | prompt_ids = mm_res.prompt_ids |
| 657 | mm_kwargs = mm_res.kwargs |
| 658 | mm_prompt_updates = mm_res.prompt_updates |
| 659 | |
| 660 | self._validate_mm_kwargs(mm_kwargs, mm_item_counts) |
| 661 | self._validate_mm_updates(mm_prompt_updates, mm_item_counts) |
| 662 | |
| 663 | # Detect use_audio_in_video from mm_kwargs |
| 664 | use_audio_in_video = False |
| 665 | if "video" in mm_kwargs: |
| 666 | for item in mm_kwargs["video"]: |
| 667 | if item and item.get("use_audio_in_video"): |
| 668 | use_audio_in_video_tensor = item["use_audio_in_video"].data |
| 669 | assert isinstance(use_audio_in_video_tensor, torch.Tensor) |
| 670 | if use_audio_in_video_tensor.numel() > 0: |
| 671 | use_audio_in_video = bool(use_audio_in_video_tensor.item()) |
| 672 | break |
| 673 | # for mutilmodality cache |
| 674 | if any(item is None for item in mm_kwargs["video"]): |
| 675 | video_token_id = self.info.get_hf_config().video_token_id |
| 676 | audio_token_id = self.info.get_hf_config().audio_token_id |
| 677 | video_audio_item_num = sum( |
| 678 | id in (video_token_id, audio_token_id) for id in prompt_ids |
| 679 | ) |
| 680 | audio_updates_num = len(mm_prompt_updates.get("audio", [])) |
| 681 | video_updates_num = len(mm_prompt_updates.get("video", [])) |
| 682 | if video_audio_item_num != video_updates_num + audio_updates_num: |
| 683 | use_audio_in_video = True |
| 684 | |
| 685 | if use_audio_in_video and "audio" in mm_prompt_updates: |
| 686 | # Filter out audio updates - they are embedded in video |
| 687 | filtered_updates = { |
| 688 | k: v for k, v in mm_prompt_updates.items() if k != "audio" |
| 689 | } |
| 690 | prompt_ids, mm_placeholders = self._apply_prompt_updates( |
| 691 | prompt_ids, |
| 692 | filtered_updates, |
| 693 | ) |
| 694 | # Derive audio placeholders from video placeholders |
| 695 | mm_placeholders = self._derive_audio_from_video_placeholders( |
| 696 | mm_placeholders, mm_prompt_updates |
| 697 | ) |
| 698 | else: |
| 699 | prompt_ids, mm_placeholders = self._apply_prompt_updates( |
| 700 | prompt_ids, |
| 701 | mm_prompt_updates, |
| 702 | ) |
| 703 | |
| 704 | self._validate_mm_placeholders(mm_placeholders, mm_item_counts) |
| 705 | |
| 706 | return prompt_ids, mm_placeholders |
| 707 | |
| 708 | @classmethod |
| 709 | def omni_get_updates_use_audio_in_video( |
| 710 | cls, |
| 711 | thinker_config: PreTrainedConfig, |
| 712 | audio_len: int, |
| 713 | video_grid_thw: list[int] | torch.Tensor, |
| 714 | video_second_per_grid_t: float, |
| 715 | ) -> list[int]: |
| 716 | """Get video prompt updates when `use_audio_in_video` is True. |
| 717 | |
| 718 | In this case, audio and vision update ids will be split into |
| 719 | chunks and interleaved (details in `_omni_get_input_positions_tensor`). |
| 720 | |
| 721 | <|video_bos|><|VIDEO|><|video_eos|> => |
| 722 | <|video_bos|><|audio_bos|>(... chunks ...)<|audio_eos|><|video_eos|> |
| 723 | """ |
| 724 | audio_token_id = thinker_config.audio_token_index |
| 725 | video_token_id = thinker_config.video_token_index |
| 726 | audio_start_token_id = thinker_config.audio_start_token_id |
| 727 | audio_end_token_id = thinker_config.audio_end_token_id |
| 728 | seconds_per_chunk = thinker_config.seconds_per_chunk |
| 729 | spatial_merge_size = thinker_config.vision_config.spatial_merge_size |
| 730 | tokens_per_second = getattr( |
| 731 | thinker_config.vision_config, "tokens_per_second", 25 |
| 732 | ) |
| 733 | |
| 734 | grid_t = video_grid_thw[0] |
| 735 | grid_h = video_grid_thw[1] |
| 736 | grid_w = video_grid_thw[2] |
| 737 | t_ntoken_per_chunk = int(tokens_per_second * seconds_per_chunk) |
| 738 | t_index = ( |
| 739 | torch.arange(grid_t) * video_second_per_grid_t * tokens_per_second |
| 740 | ).long() |
| 741 | t_index_split_chunk = split_list_into_ranges(t_index, t_ntoken_per_chunk) |
| 742 | |
| 743 | updates = [audio_start_token_id] |
| 744 | added_audio_len = 0 |
| 745 | for t_chunk in t_index_split_chunk: |
| 746 | vision_ntoken_per_chunk = ( |
| 747 | len(t_chunk) * grid_h * grid_w // (spatial_merge_size**2) |
| 748 | ) |
| 749 | updates.extend([video_token_id] * vision_ntoken_per_chunk) |
| 750 | |
| 751 | audio_chunk_size = min(t_ntoken_per_chunk, audio_len - added_audio_len) |
| 752 | updates.extend(audio_chunk_size * [audio_token_id]) |
| 753 | added_audio_len += audio_chunk_size |
| 754 | if added_audio_len < audio_len: |
| 755 | updates.extend((audio_len - added_audio_len) * [audio_token_id]) |
| 756 | updates.extend([audio_end_token_id]) |
| 757 | |
| 758 | return updates |
| 759 | |
| 760 | def _get_prompt_updates( |
| 761 | self, |
| 762 | mm_items: MultiModalDataItems, |
| 763 | hf_processor_mm_kwargs: Mapping[str, Any], |
| 764 | out_mm_kwargs: MultiModalKwargsItems, |
| 765 | ) -> Sequence[PromptUpdate]: |
| 766 | processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) |
| 767 | tokenizer = self.info.get_tokenizer() |
| 768 | assert isinstance(self.info, Qwen2VLProcessingInfo) |
| 769 | image_processor = self.info.get_image_processor(**hf_processor_mm_kwargs) |
| 770 | vocab = tokenizer.get_vocab() |
| 771 | |
| 772 | audio_token = processor.audio_token |
| 773 | image_token = processor.image_token |
| 774 | video_token = processor.video_token |
| 775 | audio_token_id = vocab[audio_token] |
| 776 | image_token_id = vocab[image_token] |
| 777 | video_token_id = vocab[video_token] |
| 778 | |
| 779 | out_mm_data = out_mm_kwargs.get_data() |
| 780 | audio_feature_lengths = out_mm_data.get("audio_feature_lengths") |
| 781 | feature_attention_mask = out_mm_data.get("feature_attention_mask") |
| 782 | if audio_feature_lengths is None and feature_attention_mask is None: |
| 783 | audio_output_lengths = [] |
| 784 | elif audio_feature_lengths is not None: |
| 785 | _, audio_output_lens = _get_feat_extract_output_lengths( |
| 786 | audio_feature_lengths |
| 787 | ) |
| 788 | audio_output_lengths = audio_output_lens.tolist() |
| 789 | elif feature_attention_mask is not None: |
| 790 | assert isinstance(feature_attention_mask, torch.Tensor) |
| 791 | _, audio_output_lens = _get_feat_extract_output_lengths( |
| 792 | feature_attention_mask.sum(-1) |
| 793 | ) |
| 794 | audio_output_lengths = audio_output_lens.tolist() |
| 795 | |
| 796 | # number of audios read from video. |
| 797 | audio_in_video_item_idx = 0 |
| 798 | |
| 799 | def get_replacement_qwen2_audio(item_idx: int): |
| 800 | item_idx += audio_in_video_item_idx |
| 801 | |
| 802 | num_features = audio_output_lengths[item_idx] |
| 803 | if num_features == 0: |
| 804 | audios = mm_items.get_items("audio", AudioProcessorItems) |
| 805 | audio = audios.get(item_idx) |
| 806 | assert audio is not None |
| 807 | raise ValueError( |
| 808 | f"The audio {audio} (len={len(audio)}) is too short " |
| 809 | "to be represented inside the model" |
| 810 | ) |
| 811 | |
| 812 | return [audio_token_id] * num_features |
| 813 | |
| 814 | def get_replacement_qwen2_vision(item_idx: int, modality: str): |
| 815 | grid_thw = out_mm_data[f"{modality}_grid_thw"][item_idx] |
| 816 | assert isinstance(grid_thw, torch.Tensor) |
| 817 | merge_length = image_processor.merge_size**2 |
| 818 | |
| 819 | token_id = image_token_id if modality == "image" else video_token_id |
| 820 | return [token_id] * (int(grid_thw.prod()) // merge_length) |
| 821 | |
| 822 | use_audio_in_video = hf_processor_mm_kwargs.get("use_audio_in_video", False) |
| 823 | thinker_config = self.info.get_hf_config() |
| 824 | |
| 825 | def get_replacement_qwen2_use_audio_in_video(item_idx: int): |
| 826 | nonlocal audio_in_video_item_idx |
| 827 | |
| 828 | audio_num_features = audio_output_lengths[audio_in_video_item_idx] |
| 829 | video_grid_thw = out_mm_data["video_grid_thw"][item_idx] |
| 830 | |
| 831 | audio_in_video_item_idx += 1 |
| 832 | |
| 833 | second_per_grid_ts = hf_processor_mm_kwargs.get("second_per_grid_ts", None) |
| 834 | if second_per_grid_ts: |
| 835 | video_second_per_grid_t = second_per_grid_ts[item_idx] |
| 836 | else: |
| 837 | video_second_per_grid_t = 1.0 |
| 838 | |
| 839 | updates = self.omni_get_updates_use_audio_in_video( |
| 840 | thinker_config=thinker_config, |
| 841 | audio_len=audio_num_features, |
| 842 | video_grid_thw=video_grid_thw, |
| 843 | video_second_per_grid_t=video_second_per_grid_t, |
| 844 | ) |
| 845 | |
| 846 | # Only video tokens should receive video embeddings |
| 847 | return PromptUpdateDetails.select_token_id( |
| 848 | seq=updates, |
| 849 | embed_token_id=video_token_id, |
| 850 | ) |
| 851 | |
| 852 | video_replacement_fn = ( |
| 853 | get_replacement_qwen2_use_audio_in_video |
| 854 | if use_audio_in_video |
| 855 | else partial(get_replacement_qwen2_vision, modality="video") |
| 856 | ) |
| 857 | |
| 858 | return [ |
| 859 | PromptReplacement( |
| 860 | modality="audio", |
| 861 | target=[audio_token_id], |
| 862 | replacement=get_replacement_qwen2_audio, |
| 863 | ), |
| 864 | PromptReplacement( |
| 865 | modality="image", |
| 866 | target=[image_token_id], |
| 867 | replacement=partial(get_replacement_qwen2_vision, modality="image"), |
| 868 | ), |
| 869 | PromptReplacement( |
| 870 | modality="video", |
| 871 | target=[video_token_id], |
| 872 | replacement=video_replacement_fn, |
| 873 | ), |
| 874 | ] |
| 875 | |
| 876 | def _postprocess_hf_mm_data( |
| 877 | self, |
| 878 | hf_data: Mapping[str, object], |
| 879 | hf_kwargs: Mapping[str, object], |
| 880 | processed_data: BatchFeature, |
| 881 | ) -> BatchFeature: |
| 882 | input_features = processed_data.pop("input_features", None) |
| 883 | feature_attention_mask = processed_data.get("feature_attention_mask", None) |
| 884 | if "input_audio_features" not in processed_data and input_features is not None: |
| 885 | if feature_attention_mask is not None: |
| 886 | input_features = input_features.permute(0, 2, 1)[ |
| 887 | feature_attention_mask.bool() |
| 888 | ].permute(1, 0) |
| 889 | processed_data["input_audio_features"] = input_features |
| 890 | if ( |
| 891 | "audio_feature_lengths" not in processed_data |
| 892 | and feature_attention_mask is not None |
| 893 | ): |
| 894 | processed_data["audio_feature_lengths"] = feature_attention_mask.sum(-1) |
| 895 | |
| 896 | video_second_per_grid = processed_data.get("video_second_per_grid", None) |
| 897 | if video_second_per_grid is not None: |
| 898 | processed_data["second_per_grid_ts"] = video_second_per_grid |
| 899 | |
| 900 | use_audio_in_video = hf_kwargs.get("use_audio_in_video", False) |
| 901 | processed_data["use_audio_in_video"] = torch.tensor(use_audio_in_video) |
| 902 | processed_data.pop("input_ids", None) |
| 903 | |
| 904 | return processed_data |
| 905 | |
| 906 | |
| 907 | class Qwen2_5OmniConditionalGenerationMixin: |
| 908 | audio_tower: nn.Module |
| 909 | visual: nn.Module |
| 910 | vllm_config: VllmConfig |
| 911 | |
| 912 | def _parse_and_validate_audio_input( |
| 913 | self, **kwargs: object |
| 914 | ) -> Qwen2_5OmniAudioFeatureInputs | None: |
| 915 | input_audio_features = kwargs.pop("input_audio_features", None) |
| 916 | audio_feature_lengths = kwargs.pop("audio_feature_lengths", None) |
| 917 | feature_attention_mask = kwargs.pop("feature_attention_mask", None) |
| 918 | if input_audio_features is None: |
| 919 | return None |
| 920 | |
| 921 | # inputs features from rust frontend is batched and padded |
| 922 | # with shape [batch_size, n_mels, padded_seq_len], different |
| 923 | # from python's shape [n_mels, batch_size * seq_len] |
| 924 | if ( |
| 925 | isinstance(input_audio_features, torch.Tensor) |
| 926 | and input_audio_features.dim() == 3 |
| 927 | ): |
| 928 | input_audio_features = unpad_and_flat_audio_features( |
| 929 | input_audio_features, audio_feature_lengths |
| 930 | ) |
| 931 | |
| 932 | return Qwen2_5OmniAudioFeatureInputs( |
| 933 | type="audio_features", |
| 934 | input_features=input_audio_features, |
| 935 | audio_feature_lengths=audio_feature_lengths, |
| 936 | feature_attention_mask=feature_attention_mask, |
| 937 | ) |
| 938 | |
| 939 | def _parse_and_validate_image_input( |
| 940 | self, |
| 941 | **kwargs: object, |
| 942 | ) -> Qwen2_5_VLImageInputs | None: |
| 943 | pixel_values = kwargs.pop("pixel_values", None) |
| 944 | image_embeds = kwargs.pop("image_embeds", None) |
| 945 | image_grid_thw = kwargs.pop("image_grid_thw", None) |
| 946 | |
| 947 | if pixel_values is None and image_embeds is None: |
| 948 | return None |
| 949 | |
| 950 | if pixel_values is not None: |
| 951 | return Qwen2_5_VLImagePixelInputs( |
| 952 | type="pixel_values", |
| 953 | pixel_values=pixel_values, |
| 954 | image_grid_thw=image_grid_thw, |
| 955 | ) |
| 956 | |
| 957 | else: |
| 958 | return Qwen2_5_VLImageEmbeddingInputs( |
| 959 | type="image_embeds", |
| 960 | image_embeds=image_embeds, |
| 961 | image_grid_thw=image_grid_thw, |
| 962 | ) |
| 963 | |
| 964 | def _parse_and_validate_video_input( |
| 965 | self, |
| 966 | **kwargs: object, |
| 967 | ) -> Qwen2_5_VLVideoInputs | None: |
| 968 | pixel_values_videos = kwargs.pop("pixel_values_videos", None) |
| 969 | video_embeds = kwargs.pop("video_embeds", None) |
| 970 | video_grid_thw = kwargs.pop("video_grid_thw", None) |
| 971 | |
| 972 | if pixel_values_videos is None and video_embeds is None: |
| 973 | return None |
| 974 | |
| 975 | if pixel_values_videos is not None: |
| 976 | return Qwen2_5_VLVideoPixelInputs( |
| 977 | type="pixel_values_videos", |
| 978 | pixel_values_videos=pixel_values_videos, |
| 979 | video_grid_thw=video_grid_thw, |
| 980 | ) |
| 981 | |
| 982 | else: |
| 983 | if not isinstance(video_embeds, torch.Tensor): |
| 984 | raise ValueError( |
| 985 | "Incorrect type of video embeddings. " |
| 986 | f"Got type: {type(video_embeds)}" |
| 987 | ) |
| 988 | return Qwen2_5_VLVideoEmbeddingInputs( |
| 989 | type="video_embeds", |
| 990 | video_embeds=video_embeds, |
| 991 | video_grid_thw=video_grid_thw, |
| 992 | ) |
| 993 | |
| 994 | def _process_audio_input( |
| 995 | self, |
| 996 | audio_input: Qwen2_5OmniAudioFeatureInputs, |
| 997 | ) -> torch.Tensor: |
| 998 | input_features = audio_input["input_features"] |
| 999 | # audio_feature_lengths is keep_on_cpu; the audio tower derives |
| 1000 | # device placement from feature_lens, so move it explicitly. |
| 1001 | audio_feature_lengths = async_tensor_h2d( |
| 1002 | audio_input["audio_feature_lengths"], input_features.device |
| 1003 | ) |
| 1004 | |
| 1005 | audio_feat_lengths, audio_output_lengths = ( |
| 1006 | self.audio_tower._get_feat_extract_output_lengths(audio_feature_lengths) |
| 1007 | ) |
| 1008 | |
| 1009 | with gpu_sync_allowed(): |
| 1010 | audio_outputs = self.audio_tower( |
| 1011 | input_features.to(self.audio_tower.dtype), |
| 1012 | feature_lens=audio_feature_lengths, |
| 1013 | aftercnn_lens=audio_feat_lengths, |
| 1014 | ) |
| 1015 | split_sizes = audio_output_lengths.tolist() |
| 1016 | return audio_outputs.last_hidden_state.split(split_sizes) |
| 1017 | |
| 1018 | def _process_image_input( |
| 1019 | self, image_input: Qwen2_5_VLImageInputs |
| 1020 | ) -> tuple[torch.Tensor, ...]: |
| 1021 | if image_input["type"] == "image_embeds": |
| 1022 | return image_input["image_embeds"].type(self.visual.dtype) |
| 1023 | |
| 1024 | grid_thw = image_input["image_grid_thw"] |
| 1025 | assert grid_thw.ndim == 2 |
| 1026 | |
| 1027 | pixel_values = image_input["pixel_values"].type(self.visual.dtype) |
| 1028 | with set_forward_context(None, self.vllm_config): |
| 1029 | image_embeds = self.visual(pixel_values, grid_thw=grid_thw) |
| 1030 | # Split concatenated embeddings for each image item. |
| 1031 | merge_size = self.visual.spatial_merge_size |
| 1032 | sizes = grid_thw.prod(-1) // merge_size // merge_size |
| 1033 | |
| 1034 | return image_embeds.split(sizes.tolist()) |
| 1035 | |
| 1036 | def _process_video_input( |
| 1037 | self, |
| 1038 | video_input: Qwen2_5_VLVideoInputs, |
| 1039 | ) -> torch.Tensor: |
| 1040 | if video_input["type"] == "video_embeds": |
| 1041 | return video_input["video_embeds"].type(self.visual.dtype) |
| 1042 | |
| 1043 | grid_thw = video_input["video_grid_thw"] |
| 1044 | assert grid_thw.ndim == 2 |
| 1045 | |
| 1046 | pixel_values_videos = video_input["pixel_values_videos"].type(self.visual.dtype) |
| 1047 | with set_forward_context(None, self.vllm_config): |
| 1048 | video_embeds = self.visual(pixel_values_videos, grid_thw=grid_thw) |
| 1049 | # Split concatenated embeddings for each video item. |
| 1050 | merge_size = self.visual.spatial_merge_size |
| 1051 | sizes = grid_thw.prod(-1) // merge_size // merge_size |
| 1052 | |
| 1053 | return video_embeds.split(sizes.tolist()) |
| 1054 | |
| 1055 | |
| 1056 | @MULTIMODAL_REGISTRY.register_processor( |
| 1057 | Qwen2_5OmniThinkerMultiModalProcessor, |
| 1058 | info=Qwen2_5OmniThinkerProcessingInfo, |
| 1059 | dummy_inputs=Qwen2_5OmniThinkerDummyInputsBuilder, |
| 1060 | ) |
| 1061 | class Qwen2_5OmniThinkerForConditionalGeneration( |
| 1062 | nn.Module, |
| 1063 | SupportsMultiModal, |
| 1064 | SupportsPP, |
| 1065 | SupportsLoRA, |
| 1066 | SupportsMRoPE, |
| 1067 | Qwen2_5OmniConditionalGenerationMixin, |
| 1068 | ): |
| 1069 | hf_to_vllm_mapper = WeightsMapper( |
| 1070 | orig_to_new_prefix={ |
| 1071 | "thinker.lm_head.": "language_model.lm_head.", |
| 1072 | "thinker.model.": "language_model.model.", |
| 1073 | "thinker.": "", |
| 1074 | "talker.": None, |
| 1075 | "token2wav.": None, |
| 1076 | } |
| 1077 | ) |
| 1078 | packed_modules_mapping = { |
| 1079 | "qkv_proj": [ |
| 1080 | "q_proj", |
| 1081 | "k_proj", |
| 1082 | "v_proj", |
| 1083 | ], |
| 1084 | "attn.qkv": [ |
| 1085 | "attn.q", |
| 1086 | "attn.k", |
| 1087 | "attn.v", |
| 1088 | ], |
| 1089 | "gate_up_proj": [ |
| 1090 | "gate_proj", |
| 1091 | "up_proj", |
| 1092 | ], |
| 1093 | } |
| 1094 | |
| 1095 | @classmethod |
| 1096 | def get_placeholder_str(cls, modality: str, i: int) -> str | None: |
| 1097 | if modality.startswith("image"): |
| 1098 | return "<|vision_start|><|IMAGE|><|vision_end|>" |
| 1099 | if modality.startswith("video"): |
| 1100 | return "<|vision_start|><|VIDEO|><|vision_end|>" |
| 1101 | if modality.startswith("audio"): |
| 1102 | return f"Audio {i}: <|audio_bos|><|AUDIO|><|audio_eos|>" |
| 1103 | |
| 1104 | raise ValueError("Only image, video or audio modality is supported") |
| 1105 | |
| 1106 | def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): |
| 1107 | super().__init__() |
| 1108 | self.vllm_config = vllm_config |
| 1109 | thinker_config: Qwen2_5OmniThinkerConfig = ( |
| 1110 | vllm_config.model_config.hf_config.thinker_config |
| 1111 | ) |
| 1112 | quant_config = vllm_config.quant_config |
| 1113 | multimodal_config = vllm_config.model_config.multimodal_config |
| 1114 | self.config = thinker_config |
| 1115 | self.multimodal_config = multimodal_config |
| 1116 | self.quant_config = quant_config |
| 1117 | |
| 1118 | # force "use_flash_attention_2=True" to audio tower to align |
| 1119 | # the results. |
| 1120 | if flash_attn is not None: |
| 1121 | audio_config = thinker_config.audio_config |
| 1122 | audio_config._attn_implementation_autoset = True |
| 1123 | audio_config._attn_implementation = "flash_attention_2" |
| 1124 | else: |
| 1125 | logger.warning( |
| 1126 | "flash_attn is not available, the model may not yield the " |
| 1127 | "exactly same result as the transformers implementation " |
| 1128 | "in the audio tower part." |
| 1129 | ) |
| 1130 | |
| 1131 | with self._mark_tower_model(vllm_config, "audio"): |
| 1132 | self.audio_tower = Qwen2_5OmniAudioEncoder(thinker_config.audio_config) |
| 1133 | |
| 1134 | with self._mark_tower_model(vllm_config, {"image", "video"}): |
| 1135 | self.visual = Qwen2_5_VisionTransformer( |
| 1136 | vision_config=thinker_config.vision_config, |
| 1137 | norm_eps=getattr(thinker_config.text_config, "rms_norm_eps", 1e-6), |
| 1138 | quant_config=quant_config, |
| 1139 | prefix=maybe_prefix(prefix, "visual"), |
| 1140 | ) |
| 1141 | |
| 1142 | with self._mark_language_model(vllm_config): |
| 1143 | self.language_model = init_vllm_registered_model( |
| 1144 | vllm_config=vllm_config, |
| 1145 | prefix=maybe_prefix(prefix, "language_model"), |
| 1146 | hf_config=thinker_config.text_config, |
| 1147 | architectures=["Qwen2ForCausalLM"], |
| 1148 | ) |
| 1149 | |
| 1150 | self.make_empty_intermediate_tensors = ( |
| 1151 | self.language_model.make_empty_intermediate_tensors |
| 1152 | ) |
| 1153 | |
| 1154 | def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict: |
| 1155 | mm_input_by_modality: dict[ |
| 1156 | str, |
| 1157 | Qwen2_5_VLImageInputs |
| 1158 | | Qwen2_5_VLVideoInputs |
| 1159 | | Qwen2_5OmniAudioFeatureInputs |
| 1160 | | None, |
| 1161 | ] = {} |
| 1162 | |
| 1163 | # Preserve the order of modalities if there are multiple of them |
| 1164 | # from the order of kwargs. |
| 1165 | for input_key in kwargs: |
| 1166 | if ( |
| 1167 | input_key in ("pixel_values", "image_embeds") |
| 1168 | and "image" not in mm_input_by_modality |
| 1169 | ): |
| 1170 | mm_input_by_modality["image"] = self._parse_and_validate_image_input( |
| 1171 | **kwargs |
| 1172 | ) |
| 1173 | if ( |
| 1174 | input_key in ("pixel_values_videos", "video_embeds") |
| 1175 | and "video" not in mm_input_by_modality |
| 1176 | ): |
| 1177 | mm_input_by_modality["video"] = self._parse_and_validate_video_input( |
| 1178 | **kwargs |
| 1179 | ) |
| 1180 | if ( |
| 1181 | input_key in ("input_audio_features") |
| 1182 | and "audio" not in mm_input_by_modality |
| 1183 | ): |
| 1184 | mm_input_by_modality["audio"] = self._parse_and_validate_audio_input( |
| 1185 | **kwargs |
| 1186 | ) |
| 1187 | return mm_input_by_modality |
| 1188 | |
| 1189 | def _get_audio_for_video_mapping( |
| 1190 | self, mm_features: list[MultiModalFeatureSpec] |
| 1191 | ) -> tuple[dict[int, int], set[int]]: |
| 1192 | """Map video offset -> paired audio_feature_length for use_audio_in_video. |
| 1193 | |
| 1194 | When use_audio_in_video=True, audio is interleaved within video chunks. |
| 1195 | The pairing is based on feature order in mm_features. |
| 1196 | |
| 1197 | Returns: |
| 1198 | Tuple of (video_offset -> audio_feature_length mapping, |
| 1199 | set of paired audio offsets to skip) |
| 1200 | |
| 1201 | """ |
| 1202 | videos_with_audio = [ |
| 1203 | f |
| 1204 | for f in mm_features |
| 1205 | if f.modality == "video" |
| 1206 | and f.data is not None |
| 1207 | and f.data.get("use_audio_in_video") |
| 1208 | and _get_feature_tensor(f, "use_audio_in_video").item() |
| 1209 | ] |
| 1210 | audios = [f for f in mm_features if f.modality == "audio"] |
| 1211 | |
| 1212 | # Pair videos with audio features (assumes matching order) |
| 1213 | mapping: dict[int, int] = {} |
| 1214 | paired_audio_offsets: set[int] = set() |
| 1215 | for i, video_f in enumerate(videos_with_audio): |
| 1216 | if i < len(audios): |
| 1217 | audio_len = int( |
| 1218 | _get_feature_tensor(audios[i], "audio_feature_lengths").item() |
| 1219 | ) |
| 1220 | mapping[video_f.mm_position.offset] = audio_len |
| 1221 | paired_audio_offsets.add(audios[i].mm_position.offset) |
| 1222 | return mapping, paired_audio_offsets |
| 1223 | |
| 1224 | def _compute_audio_token_count(self, audio_feature_length: int) -> int: |
| 1225 | """Compute audio tokens from feature length.""" |
| 1226 | return ((audio_feature_length - 1) // 2 + 1 - 2) // 2 + 1 |
| 1227 | |
| 1228 | def iter_mm_features( |
| 1229 | self, mm_features: list[MultiModalFeatureSpec] |
| 1230 | ) -> Iterator[tuple[int, str, dict[str, Any]]]: |
| 1231 | """Iterate over multimodal features sorted by position offset. |
| 1232 | |
| 1233 | Yields: (offset, modality, feature_data) where feature_data contains: |
| 1234 | - image: {"grid_t", "grid_h", "grid_w", "t_factor"} |
| 1235 | - video: {"grid_t", "grid_h", "grid_w", "t_factor", |
| 1236 | "use_audio_in_video", "audio_feature_length"} |
| 1237 | - audio: {"audio_feature_length"} |
| 1238 | """ |
| 1239 | thinker_config = self.config |
| 1240 | spatial_merge_size = thinker_config.vision_config.spatial_merge_size |
| 1241 | tokens_per_second = getattr( |
| 1242 | thinker_config.vision_config, "tokens_per_second", 25 |
| 1243 | ) |
| 1244 | |
| 1245 | # Sort features by offset first, then pair audio with video |
| 1246 | sorted_features = sorted(mm_features, key=lambda f: f.mm_position.offset) |
| 1247 | audio_for_video, paired_audio_offsets = self._get_audio_for_video_mapping( |
| 1248 | sorted_features |
| 1249 | ) |
| 1250 | |
| 1251 | for mm_feature in sorted_features: |
| 1252 | offset = mm_feature.mm_position.offset |
| 1253 | modality = mm_feature.modality |
| 1254 | assert mm_feature.data is not None |
| 1255 | |
| 1256 | if modality == "image": |
| 1257 | t, h, w = _get_feature_tensor(mm_feature, "image_grid_thw").tolist() |
| 1258 | yield ( |
| 1259 | offset, |
| 1260 | "image", |
| 1261 | { |
| 1262 | "grid_t": t, |
| 1263 | "grid_h": h // spatial_merge_size, |
| 1264 | "grid_w": w // spatial_merge_size, |
| 1265 | "t_factor": 1.0 * tokens_per_second, |
| 1266 | }, |
| 1267 | ) |
| 1268 | elif modality == "video": |
| 1269 | t, h, w = _get_feature_tensor(mm_feature, "video_grid_thw").tolist() |
| 1270 | second_per_grid_ts = 1.0 |
| 1271 | if mm_feature.data.get("second_per_grid_ts"): |
| 1272 | second_per_grid_ts = _get_feature_tensor( |
| 1273 | mm_feature, "second_per_grid_ts" |
| 1274 | ).item() |
| 1275 | use_audio_in_video = False |
| 1276 | if mm_feature.data.get("use_audio_in_video"): |
| 1277 | use_audio_in_video = bool( |
| 1278 | _get_feature_tensor(mm_feature, "use_audio_in_video").item() |
| 1279 | ) |
| 1280 | |
| 1281 | yield ( |
| 1282 | offset, |
| 1283 | "video", |
| 1284 | { |
| 1285 | "grid_t": t, |
| 1286 | "grid_h": h // spatial_merge_size, |
| 1287 | "grid_w": w // spatial_merge_size, |
| 1288 | "t_factor": second_per_grid_ts * tokens_per_second, |
| 1289 | "use_audio_in_video": use_audio_in_video, |
| 1290 | "audio_feature_length": audio_for_video.get(offset), |
| 1291 | }, |
| 1292 | ) |
| 1293 | elif modality == "audio": |
| 1294 | # Skip audio that's paired with video (handled in video case) |
| 1295 | if offset not in paired_audio_offsets: |
| 1296 | audio_len = _get_feature_tensor( |
| 1297 | mm_feature, "audio_feature_lengths" |
| 1298 | ).item() |
| 1299 | yield offset, "audio", {"audio_feature_length": audio_len} |
| 1300 | |
| 1301 | def _compute_interleaved_positions( |
| 1302 | self, start_idx: int, data: dict[str, Any] |
| 1303 | ) -> tuple[np.ndarray, int]: |
| 1304 | """Compute positions for interleaved video+audio chunks. |
| 1305 | |
| 1306 | Returns: (position_ids, total_token_count) |
| 1307 | """ |
| 1308 | grid_t = data["grid_t"] |
| 1309 | grid_h = data["grid_h"] |
| 1310 | grid_w = data["grid_w"] |
| 1311 | t_factor = data["t_factor"] |
| 1312 | audio_len = data["audio_feature_length"] |
| 1313 | |
| 1314 | thinker_config = self.config |
| 1315 | tokens_per_second = getattr( |
| 1316 | thinker_config.vision_config, "tokens_per_second", 25 |
| 1317 | ) |
| 1318 | seconds_per_chunk = thinker_config.seconds_per_chunk |
| 1319 | t_ntoken_per_chunk = int(tokens_per_second * seconds_per_chunk) |
| 1320 | |
| 1321 | # Temporal indices with scaling |
| 1322 | t_index = (np.arange(grid_t) * t_factor).astype(np.int64) |
| 1323 | |
| 1324 | # Split temporal indices into chunks |
| 1325 | t_index_split_chunk: list[list[int]] = [ |
| 1326 | [] for _ in range((int(t_index.max()) // t_ntoken_per_chunk) + 1) |
| 1327 | ] |
| 1328 | for t_val in t_index: |
| 1329 | idx = int(t_val) // t_ntoken_per_chunk |
| 1330 | t_index_split_chunk[idx].append(int(t_val)) |
| 1331 | |
| 1332 | pure_audio_len = self._compute_audio_token_count(audio_len) |
| 1333 | added_audio_len = 0 |
| 1334 | pos_ids_list: list[np.ndarray] = [] |
| 1335 | audio_start_idx = start_idx |
| 1336 | |
| 1337 | for t_chunk in t_index_split_chunk: |
| 1338 | if not t_chunk: |
| 1339 | continue |
| 1340 | |
| 1341 | chunk_t = len(t_chunk) |
| 1342 | |
| 1343 | # Build vision positions for this chunk |
| 1344 | h_indices = np.tile( |
| 1345 | np.arange(grid_h).reshape(1, -1, 1), (chunk_t, 1, grid_w) |
| 1346 | ).flatten() |
| 1347 | w_indices = np.tile( |
| 1348 | np.arange(grid_w).reshape(1, 1, -1), (chunk_t, grid_h, 1) |
| 1349 | ).flatten() |
| 1350 | t_indices = np.repeat(np.array(t_chunk), grid_h * grid_w) |
| 1351 | |
| 1352 | vision_pos = np.stack([t_indices, h_indices, w_indices]) + start_idx |
| 1353 | pos_ids_list.append(vision_pos) |
| 1354 | |
| 1355 | # Audio tokens for this chunk |
| 1356 | audio_chunk_size = min(t_ntoken_per_chunk, pure_audio_len - added_audio_len) |
| 1357 | if audio_chunk_size > 0: |
| 1358 | audio_pos = ( |
| 1359 | np.broadcast_to(np.arange(audio_chunk_size), (3, audio_chunk_size)) |
| 1360 | + audio_start_idx |
| 1361 | ) |
| 1362 | pos_ids_list.append(audio_pos) |
| 1363 | audio_start_idx = audio_start_idx + audio_chunk_size |
| 1364 | added_audio_len += audio_chunk_size |
| 1365 | |
| 1366 | # Handle remaining audio that doesn't fit in chunks |
| 1367 | if added_audio_len < pure_audio_len: |
| 1368 | remaining = pure_audio_len - added_audio_len |
| 1369 | remaining_audio_pos = ( |
| 1370 | np.broadcast_to(np.arange(remaining), (3, remaining)) + audio_start_idx |
| 1371 | ) |
| 1372 | pos_ids_list.append(remaining_audio_pos) |
| 1373 | |
| 1374 | # Calculate total token count |
| 1375 | vision_tokens = grid_t * grid_h * grid_w |
| 1376 | total_tokens = vision_tokens + pure_audio_len |
| 1377 | |
| 1378 | return np.concatenate(pos_ids_list, axis=1), total_tokens |
| 1379 | |
| 1380 | def get_mrope_input_positions( |
| 1381 | self, |
| 1382 | input_tokens: list[int], |
| 1383 | mm_features: list[MultiModalFeatureSpec], |
| 1384 | ) -> tuple[torch.Tensor, int]: |
| 1385 | """Compute M-RoPE input positions using mm_features directly. |
| 1386 | |
| 1387 | Example for use_audio_in_video case: |
| 1388 | (V_i are vision position ids, A_i are audio position ids) |
| 1389 | |
| 1390 | |V_1 ... V_n|A_1 ... A_n|V_n+1 ... V_2n|A_n+1 ... A_2n|... |
| 1391 | |vision chunk 1|audio chunk 1|vision chunk 2|audio chunk 2 |... |
| 1392 | """ |
| 1393 | llm_pos_ids_list: list[np.ndarray] = [] |
| 1394 | st = 0 |
| 1395 | |
| 1396 | for offset, modality, data in self.iter_mm_features(mm_features): |
| 1397 | # Add text segment before this feature |
| 1398 | text_len = offset - st |
| 1399 | st_idx = int(llm_pos_ids_list[-1].max()) + 1 if llm_pos_ids_list else 0 |
| 1400 | if text_len > 0: |
| 1401 | llm_pos_ids_list.append( |
| 1402 | np.broadcast_to(np.arange(text_len), (3, text_len)) + st_idx |
| 1403 | ) |
| 1404 | st_idx += text_len |
| 1405 | |
| 1406 | if modality == "audio": |
| 1407 | # Standalone audio positions |
| 1408 | audio_tokens = self._compute_audio_token_count( |
| 1409 | data["audio_feature_length"] |
| 1410 | ) |
| 1411 | llm_pos_ids_list.append( |
| 1412 | np.broadcast_to(np.arange(audio_tokens), (3, audio_tokens)) + st_idx |
| 1413 | ) |
| 1414 | st = offset + audio_tokens |
| 1415 | |
| 1416 | elif modality == "image": |
| 1417 | # Image uses np.indices like Qwen2-VL |
| 1418 | grid_t = data["grid_t"] |
| 1419 | grid_h = data["grid_h"] |
| 1420 | grid_w = data["grid_w"] |
| 1421 | t_factor = data["t_factor"] |
| 1422 | |
| 1423 | grid_indices = np.indices((grid_t, grid_h, grid_w)) |
| 1424 | if t_factor != 1.0: |
| 1425 | grid_indices[0] = (grid_indices[0] * t_factor).astype(np.int64) |
| 1426 | llm_pos_ids_list.append(grid_indices.reshape(3, -1) + st_idx) |
| 1427 | st = offset + grid_t * grid_h * grid_w |
| 1428 | |
| 1429 | elif modality == "video": |
| 1430 | grid_t = data["grid_t"] |
| 1431 | grid_h = data["grid_h"] |
| 1432 | grid_w = data["grid_w"] |
| 1433 | t_factor = data["t_factor"] |
| 1434 | |
| 1435 | if not data["use_audio_in_video"]: |
| 1436 | # Simple video (same as Qwen2-VL) |
| 1437 | grid_indices = np.indices((grid_t, grid_h, grid_w)) |
| 1438 | if t_factor != 1.0: |
| 1439 | grid_indices[0] = (grid_indices[0] * t_factor).astype(np.int64) |
| 1440 | llm_pos_ids_list.append(grid_indices.reshape(3, -1) + st_idx) |
| 1441 | st = offset + grid_t * grid_h * grid_w |
| 1442 | else: |
| 1443 | # Interleaved video+audio |
| 1444 | pos_ids, token_count = self._compute_interleaved_positions( |
| 1445 | st_idx, data |
| 1446 | ) |
| 1447 | llm_pos_ids_list.append(pos_ids) |
| 1448 | st = offset + token_count |
| 1449 | |
| 1450 | # Add trailing text |
| 1451 | if st < len(input_tokens): |
| 1452 | st_idx = int(llm_pos_ids_list[-1].max()) + 1 if llm_pos_ids_list else 0 |
| 1453 | text_len = len(input_tokens) - st |
| 1454 | llm_pos_ids_list.append( |
| 1455 | np.broadcast_to(np.arange(text_len), (3, text_len)) + st_idx |
| 1456 | ) |
| 1457 | |
| 1458 | llm_positions = np.concatenate(llm_pos_ids_list, axis=1).reshape(3, -1) |
| 1459 | mrope_position_delta = int(llm_positions.max()) + 1 - len(input_tokens) |
| 1460 | |
| 1461 | return torch.from_numpy(llm_positions), mrope_position_delta |
| 1462 | |
| 1463 | def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings: |
| 1464 | mm_input_by_modality = self._parse_and_validate_multimodal_inputs(**kwargs) |
| 1465 | if not mm_input_by_modality: |
| 1466 | return [] |
| 1467 | |
| 1468 | # The result multimodal_embeddings is tuple of tensors, with each |
| 1469 | # tensor corresponding to a multimodal data item (image or video). |
| 1470 | multimodal_embeddings: tuple[torch.Tensor, ...] = () |
| 1471 | |
| 1472 | # NOTE: It is important to iterate over the keys in this dictionary |
| 1473 | # to preserve the order of the modalities. |
| 1474 | for modality in mm_input_by_modality: |
| 1475 | multimodal_input = mm_input_by_modality[modality] |
| 1476 | if modality == "image": |
| 1477 | image_embeddings = self._process_image_input(multimodal_input) |
| 1478 | multimodal_embeddings += tuple(image_embeddings) |
| 1479 | if modality == "video": |
| 1480 | video_embeddings = self._process_video_input(multimodal_input) |
| 1481 | multimodal_embeddings += tuple(video_embeddings) |
| 1482 | if modality == "audio": |
| 1483 | audio_embeddings = self._process_audio_input(multimodal_input) |
| 1484 | multimodal_embeddings += tuple(audio_embeddings) |
| 1485 | return multimodal_embeddings |
| 1486 | |
| 1487 | def embed_input_ids( |
| 1488 | self, |
| 1489 | input_ids: torch.Tensor, |
| 1490 | multimodal_embeddings: MultiModalEmbeddings | None = None, |
| 1491 | *, |
| 1492 | is_multimodal: torch.Tensor | None = None, |
| 1493 | ) -> torch.Tensor: |
| 1494 | if multimodal_embeddings is None or is_multimodal is None: |
| 1495 | return super().embed_input_ids(input_ids) |
| 1496 | |
| 1497 | if len(multimodal_embeddings) == 0: |
| 1498 | return super().embed_input_ids( |
| 1499 | input_ids, |
| 1500 | multimodal_embeddings=multimodal_embeddings, |
| 1501 | is_multimodal=is_multimodal, |
| 1502 | ) |
| 1503 | |
| 1504 | # Check for audio-in-video: interleaved video and audio tokens |
| 1505 | # in the multimodal region. Only use the interleaved path when |
| 1506 | # needed; otherwise fall back to the default parent implementation. |
| 1507 | video_token_id = self.config.video_token_index |
| 1508 | audio_token_id = self.config.audio_token_index |
| 1509 | |
| 1510 | with gpu_sync_allowed(): |
| 1511 | input_ids_cpu = input_ids.cpu() |
| 1512 | is_video = is_multimodal & (input_ids_cpu == video_token_id) |
| 1513 | is_audio = is_multimodal & (input_ids_cpu == audio_token_id) |
| 1514 | |
| 1515 | num_video = is_video.sum().item() |
| 1516 | num_audio = is_audio.sum().item() |
| 1517 | |
| 1518 | if check_interleaved_audio_video(is_video, is_audio, num_video, num_audio): |
| 1519 | inputs_embeds = self._embed_text_input_ids( |
| 1520 | input_ids, |
| 1521 | self.get_language_model().embed_input_ids, |
| 1522 | is_multimodal=is_multimodal, |
| 1523 | ) |
| 1524 | return merge_interleaved_embeddings( |
| 1525 | inputs_embeds, |
| 1526 | multimodal_embeddings, |
| 1527 | is_video, |
| 1528 | is_audio, |
| 1529 | is_multimodal, |
| 1530 | ) |
| 1531 | |
| 1532 | # Default: standard merge (no interleaving), same as parent class |
| 1533 | return super().embed_input_ids( |
| 1534 | input_ids, |
| 1535 | multimodal_embeddings=multimodal_embeddings, |
| 1536 | is_multimodal=is_multimodal, |
| 1537 | ) |
| 1538 | |
| 1539 | def forward( |
| 1540 | self, |
| 1541 | input_ids: torch.Tensor | None, |
| 1542 | positions: torch.Tensor, |
| 1543 | intermediate_tensors: IntermediateTensors | None = None, |
| 1544 | inputs_embeds: torch.Tensor | None = None, |
| 1545 | **kwargs: object, |
| 1546 | ) -> torch.Tensor | IntermediateTensors: |
| 1547 | if intermediate_tensors is not None: |
| 1548 | inputs_embeds = None |
| 1549 | |
| 1550 | hidden_states = self.language_model.model( |
| 1551 | input_ids, positions, intermediate_tensors, inputs_embeds=inputs_embeds |
| 1552 | ) |
| 1553 | return hidden_states |
| 1554 | |
| 1555 | def compute_logits( |
| 1556 | self, |
| 1557 | hidden_states: torch.Tensor, |
| 1558 | ) -> torch.Tensor | None: |
| 1559 | return self.language_model.compute_logits(hidden_states) |
| 1560 | |
| 1561 | def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: |
| 1562 | loader = AutoWeightsLoader(self) |
| 1563 | return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) |
| 1564 | |
| 1565 | def get_mm_mapping(self) -> MultiModelKeys: |
| 1566 | """Get the module prefix in multimodal models""" |
| 1567 | return MultiModelKeys.from_string_field( |
| 1568 | language_model="language_model", |
| 1569 | connector="merger.", |
| 1570 | tower_model=["visual.", "audio_tower."], |
| 1571 | ) |