@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 193069d129

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 9, 23:00 UTC

[5/N] Initialize MM components in context managers (Q-Z) (#32695)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Jan 21, 2026 at 03:10 UTC 193069d129414f8f70218f8fb88fb92c75f17ce5

9 files changed +177 -167

vllm/model_executor/models/qwen2_audio.py

+13 -15

@@ -334,20 +334,21 @@ class Qwen2AudioForConditionalGeneration(nn.Module, SupportsMultiModal, Supports

334 multimodal_config = vllm_config.model_config.multimodal_config

335 self.config = config

336 self.multimodal_config = multimodal_config

337 -

338 - self.audio_tower = Qwen2AudioEncoder(config.audio_config)

339 - self.multi_modal_projector = Qwen2AudioMultiModalProjector(

340 - config.audio_config.d_model, config.text_config.hidden_size

341 - )

342 -

337 self.quant_config = quant_config

338

345 - self.language_model = init_vllm_registered_model(

346 - vllm_config=vllm_config,

347 - hf_config=config.text_config,

348 - prefix=maybe_prefix(prefix, "language_model"),

349 - architectures=["Qwen2ForCausalLM"],

350 - )

339 + with self._mark_tower_model(vllm_config, "audio"):

340 + self.audio_tower = Qwen2AudioEncoder(config.audio_config)

341 + self.multi_modal_projector = Qwen2AudioMultiModalProjector(

342 + config.audio_config.d_model, config.text_config.hidden_size

343 + )

344 +

345 + with self._mark_language_model(vllm_config):

346 + self.language_model = init_vllm_registered_model(

347 + vllm_config=vllm_config,

348 + hf_config=config.text_config,

349 + prefix=maybe_prefix(prefix, "language_model"),

350 + architectures=["Qwen2ForCausalLM"],

351 + )

352

353 self.make_empty_intermediate_tensors = (

354 self.language_model.make_empty_intermediate_tensors

@@ -441,9 +442,6 @@ class Qwen2AudioForConditionalGeneration(nn.Module, SupportsMultiModal, Supports

442 masked_audio_features, audio_output_lengths.flatten().tolist()

443 )

444

444 - def get_language_model(self) -> torch.nn.Module:

445 - return self.language_model

446 -

445 def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:

446 audio_input = self._parse_and_validate_audio_input(**kwargs)

447 if audio_input is None:

vllm/model_executor/models/qwen3_omni_moe_thinker.py

+52 -45

@@ -1612,32 +1612,14 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(

1612 multimodal_config = vllm_config.model_config.multimodal_config

1613 self.config = thinker_config

1614 self.multimodal_config = multimodal_config

1615 -

1616 - self.audio_tower = Qwen3OmniMoeAudioEncoder(

1617 - thinker_config.audio_config,

1618 - multimodal_config=multimodal_config,

1619 - prefix=maybe_prefix(prefix, "audio_tower"),

1620 - )

1621 -

1622 - self.visual = Qwen3Omni_VisionTransformer(

1623 - vision_config=thinker_config.vision_config,

1624 - norm_eps=getattr(thinker_config.text_config, "rms_norm_eps", 1e-6),

1625 - quant_config=quant_config,

1626 - prefix=maybe_prefix(prefix, "visual"),

1627 - multimodal_config=multimodal_config,

1628 - )

1615 self.quant_config = quant_config

1616

1631 - self.language_model = Qwen3MoeLLMForCausalLM(

1632 - vllm_config=vllm_config.with_hf_config(

1633 - thinker_config.text_config, architectures=["Qwen3MoeForCausalLM"]

1634 - ),

1635 - prefix=maybe_prefix(prefix, "language_model"),

1636 - )

1637 -

1638 - self.make_empty_intermediate_tensors = (

1639 - self.language_model.make_empty_intermediate_tensors

1640 - )

1617 + with self._mark_tower_model(vllm_config, "audio"):

1618 + self.audio_tower = Qwen3OmniMoeAudioEncoder(

1619 + thinker_config.audio_config,

1620 + multimodal_config=multimodal_config,

1621 + prefix=maybe_prefix(prefix, "audio_tower"),

1622 + )

1623

1624 self.use_deepstack = hasattr(

1625 thinker_config.vision_config, "deepstack_visual_indexes"

@@ -1647,22 +1629,48 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(

1629 if self.use_deepstack

1630 else 0

1631 )

1650 - # register buffer for deepstack

1651 - self.deepstack_input_embeds = (

1652 - [

1653 - torch.zeros(

1654 - vllm_config.scheduler_config.max_num_batched_tokens,

1655 - thinker_config.text_config.hidden_size,

1656 - )

1657 - for _ in range(self.deepstack_num_level)

1658 - ]

1659 - if self.use_deepstack

1660 - else None

1661 - )

1632 self.visual_dim = thinker_config.vision_config.out_hidden_size

1633 self.multiscale_dim = self.visual_dim * self.deepstack_num_level

1634

1665 - def _get_deepstack_input_embeds(self, num_tokens: int) -> IntermediateTensors:

1635 + with self._mark_tower_model(vllm_config, {"image", "video"}):

1636 + self.visual = Qwen3Omni_VisionTransformer(

1637 + vision_config=thinker_config.vision_config,

1638 + norm_eps=getattr(thinker_config.text_config, "rms_norm_eps", 1e-6),

1639 + quant_config=quant_config,

1640 + prefix=maybe_prefix(prefix, "visual"),

1641 + multimodal_config=multimodal_config,

1642 + )

1643 +

1644 + # register buffer for deepstack

1645 + if self.use_deepstack:

1646 + self.deepstack_input_embeds = [

1647 + torch.zeros(

1648 + vllm_config.scheduler_config.max_num_batched_tokens,

1649 + thinker_config.text_config.hidden_size,

1650 + )

1651 + for _ in range(self.deepstack_num_level)

1652 + ]

1653 +

1654 + with self._mark_language_model(vllm_config):

1655 + self.language_model = Qwen3MoeLLMForCausalLM(

1656 + vllm_config=vllm_config.with_hf_config(

1657 + thinker_config.text_config,

1658 + architectures=["Qwen3MoeForCausalLM"],

1659 + ),

1660 + prefix=maybe_prefix(prefix, "language_model"),

1661 + )

1662 +

1663 + self.make_empty_intermediate_tensors = (

1664 + self.language_model.make_empty_intermediate_tensors

1665 + )

1666 +

1667 + def _get_deepstack_input_embeds(

1668 + self,

1669 + num_tokens: int,

1670 + ) -> IntermediateTensors | None:

1671 + if not getattr(self, "deepstack_input_embeds", None):

1672 + return None # If vision tower is skipped

1673 +

1674 # get deepstack_input_embeds from buffer, and clear the buffer

1675 return IntermediateTensors(

1676 {

@@ -1674,6 +1682,9 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(

1682 )

1683

1684 def _set_deepstack_input_embeds(self, deepstack_input_embeds: torch.Tensor) -> None:

1685 + if not getattr(self, "deepstack_input_embeds", None):

1686 + return

1687 +

1688 # set deepstack_input_embeds to buffer

1689 num_tokens = deepstack_input_embeds.size(1)

1690 if num_tokens > self.deepstack_input_embeds[0].size(0):

@@ -1692,6 +1703,9 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(

1703 )

1704

1705 def _clear_deepstack_input_embeds(self, num_tokens: int) -> None:

1706 + if not getattr(self, "deepstack_input_embeds", None):

1707 + return

1708 +

1709 # clear deepstack_input_embeds in buffer

1710 if num_tokens > 0:

1711 for idx in range(self.deepstack_num_level):

@@ -1726,9 +1740,6 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(

1740 )

1741 return mm_input_by_modality

1742

1729 - def get_language_model(self) -> torch.nn.Module:

1730 - return self.language_model

1731 -

1743 def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings | None:

1744 mm_input_by_modality = self._parse_and_validate_multimodal_inputs(**kwargs)

1745 if not mm_input_by_modality:

@@ -1844,11 +1855,7 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(

1855 if intermediate_tensors is not None:

1856 inputs_embeds = None

1857

1847 - if (

1848 - self.use_deepstack

1849 - and inputs_embeds is not None

1850 - and get_pp_group().is_first_rank

1851 - ):

1858 + if inputs_embeds is not None and get_pp_group().is_first_rank:

1859 deepstack_input_embeds = self._get_deepstack_input_embeds(

1860 inputs_embeds.size(0)

1861 )

vllm/model_executor/models/qwen3_vl.py

+14 -6

@@ -1321,7 +1321,13 @@ class Qwen3VLForConditionalGeneration(

1321 num_layers = len(self.language_model.model.layers)

1322 return (2, num_layers // 2, num_layers - 3)

1323

1324 - def _get_deepstack_input_embeds(self, num_tokens: int) -> IntermediateTensors:

1324 + def _get_deepstack_input_embeds(

1325 + self,

1326 + num_tokens: int,

1327 + ) -> IntermediateTensors | None:

1328 + if not getattr(self, "deepstack_input_embeds", None):

1329 + return None # If vision tower is skipped

1330 +

1331 # get deepstack_input_embeds from buffer, and clear the buffer

1332 return IntermediateTensors(

1333 {

@@ -1333,6 +1339,9 @@ class Qwen3VLForConditionalGeneration(

1339 )

1340

1341 def _set_deepstack_input_embeds(self, deepstack_input_embeds: torch.Tensor) -> None:

1342 + if not getattr(self, "deepstack_input_embeds", None):

1343 + return

1344 +

1345 # set deepstack_input_embeds to buffer

1346 num_tokens = deepstack_input_embeds.size(1)

1347 if num_tokens > self.deepstack_input_embeds[0].size(0):

@@ -1351,6 +1360,9 @@ class Qwen3VLForConditionalGeneration(

1360 )

1361

1362 def _clear_deepstack_input_embeds(self, num_tokens: int) -> None:

1363 + if not getattr(self, "deepstack_input_embeds", None):

1364 + return

1365 +

1366 # clear deepstack_input_embeds in buffer

1367 if num_tokens > 0:

1368 for idx in range(self.deepstack_num_level):

@@ -2037,11 +2049,7 @@ class Qwen3VLForConditionalGeneration(

2049 if intermediate_tensors is not None:

2050 inputs_embeds = None

2051

2040 - if (

2041 - self.use_deepstack

2042 - and inputs_embeds is not None

2043 - and get_pp_group().is_first_rank

2044 - ):

2052 + if inputs_embeds is not None and get_pp_group().is_first_rank:

2053 deepstack_input_embeds = self._get_deepstack_input_embeds(

2054 inputs_embeds.size(0)

2055 )

vllm/model_executor/models/radio.py

-1

@@ -620,7 +620,6 @@ class RadioInternVisionModel(nn.Module):

620 x: torch.Tensor,

621 imgs_sizes: torch.Tensor | None = None,

622 ) -> torch.FloatTensor:

623 - assert self.patch_generator is not None

623 hidden_states = self.patch_generator(x, imgs_sizes=imgs_sizes)

624 attn_mask = None

625 if imgs_sizes is not None and len(imgs_sizes) > 1:

vllm/model_executor/models/siglip.py

+14 -15

@@ -1033,20 +1033,22 @@ class SiglipEmbeddingModel(nn.Module, SupportsMultiModal, SupportsQuant):

1033

1034 self.text_embed_dim = text_config.hidden_size

1035 self.vision_embed_dim = vision_config.hidden_size

1036 + self.text_projection_size = text_config.projection_size

1037

1037 - self.text_model = SiglipTextTransformer(

1038 - text_config,

1039 - quant_config=quant_config,

1040 - prefix=maybe_prefix(prefix, "text_model"),

1041 - )

1042 - self.vision_model = SiglipVisionTransformer(

1043 - vision_config,

1044 - quant_config=quant_config,

1045 - multimodal_config=multimodal_config,

1046 - prefix=maybe_prefix(prefix, "vision_model"),

1047 - )

1038 + with self._mark_language_model(vllm_config):

1039 + self.text_model = SiglipTextTransformer(

1040 + text_config,

1041 + quant_config=quant_config,

1042 + prefix=maybe_prefix(prefix, "text_model"),

1043 + )

1044

1049 - self.text_projection_size = text_config.projection_size

1045 + with self._mark_tower_model(vllm_config, "image"):

1046 + self.vision_model = SiglipVisionTransformer(

1047 + vision_config,

1048 + quant_config=quant_config,

1049 + multimodal_config=multimodal_config,

1050 + prefix=maybe_prefix(prefix, "vision_model"),

1051 + )

1052

1053 pooler_config = vllm_config.model_config.pooler_config

1054 assert pooler_config is not None

@@ -1155,9 +1157,6 @@ class SiglipEmbeddingModel(nn.Module, SupportsMultiModal, SupportsQuant):

1157

1158 return self.get_image_features(pixel_values)

1159

1158 - def get_language_model(self) -> torch.nn.Module:

1159 - return self.text_model

1160 -

1160 def _embed_text_input_ids(

1161 self,

1162 input_ids: torch.Tensor,

vllm/model_executor/models/skyworkr1v.py

+18 -21

@@ -674,24 +674,26 @@ class SkyworkR1VChatModel(nn.Module, SupportsMultiModal, SupportsPP):

674 self.downsample_ratio = config.downsample_ratio

675 self.ps_version = config.ps_version

676

677 - self.llm_arch_name = config.text_config.architectures[0]

678 - self.is_mono = self.llm_arch_name == "SkyworkLM2VEForCausalLM"

679 - self.vision_model = self._init_vision_model(

680 - config,

681 - quant_config=quant_config,

682 - is_mono=self.is_mono,

683 - prefix=maybe_prefix(prefix, "vision_model"),

684 - )

677 + llm_arch_name = config.text_config.architectures[0]

678 + self.is_mono = llm_arch_name == "SkyworkLM2VEForCausalLM"

679

686 - self.language_model = init_vllm_registered_model(

687 - vllm_config=vllm_config,

688 - hf_config=config.text_config,

689 - prefix=maybe_prefix(prefix, "language_model"),

690 - )

680 + with self._mark_tower_model(vllm_config, "image"):

681 + self.vision_model = self._init_vision_model(

682 + config,

683 + quant_config=quant_config,

684 + is_mono=self.is_mono,

685 + prefix=maybe_prefix(prefix, "vision_model"),

686 + )

687 + self.mlp1 = self._init_mlp1(

688 + config, quant_config, prefix=maybe_prefix(prefix, "mlp1")

689 + )

690

692 - self.mlp1 = self._init_mlp1(

693 - config, quant_config, prefix=maybe_prefix(prefix, "mlp1")

694 - )

691 + with self._mark_language_model(vllm_config):

692 + self.language_model = init_vllm_registered_model(

693 + vllm_config=vllm_config,

694 + hf_config=config.text_config,

695 + prefix=maybe_prefix(prefix, "language_model"),

696 + )

697

698 self.img_context_token_id = None

699 self.visual_token_mask = None

@@ -838,8 +840,6 @@ class SkyworkR1VChatModel(nn.Module, SupportsMultiModal, SupportsPP):

840 if image_input["type"] == "image_embeds":

841 return image_input["data"]

842

841 - assert self.vision_model is not None

842 -

843 image_embeds = self.extract_feature(image_input["pixel_values_flat"])

844

845 num_patches = image_input["num_patches"]

@@ -867,9 +867,6 @@ class SkyworkR1VChatModel(nn.Module, SupportsMultiModal, SupportsPP):

867 else:

868 self.visual_token_mask = None

869

870 - def get_language_model(self) -> torch.nn.Module:

871 - return self.language_model

872 -

870 def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:

871 image_input = self._parse_and_validate_image_input(**kwargs)

872 if image_input is None:

vllm/model_executor/models/tarsier.py

+37 -36

@@ -423,38 +423,43 @@ class TarsierForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP)

423 multimodal_config = vllm_config.model_config.multimodal_config

424

425 self.config = config # Storing the Tarsier-specific HF config

426 - self.vision_tower = init_vision_tower_for_tarsier(

427 - config,

428 - quant_config=quant_config,

429 - multimodal_config=multimodal_config,

430 - require_post_norm=False,

431 - prefix=maybe_prefix(prefix, "vision_tower"),

432 - )

433 - projector_bias = getattr(config, "multimodal_projector_bias", True)

426

435 - self.multi_modal_projector = TarsierMultiModalProjector(

436 - vision_hidden_size=config.vision_config.hidden_size,

437 - text_hidden_size=config.text_config.hidden_size,

438 - projector_hidden_act=config.projector_hidden_act,

439 - multimodal_projector_bias=projector_bias,

440 - quant_config=quant_config,

441 - prefix=maybe_prefix(prefix, "multi_modal_projector"),

442 - )

443 - self.language_model = init_vllm_registered_model(

444 - vllm_config=vllm_config,

445 - hf_config=config.text_config, # Use text_config from Tarsier's main config

446 - prefix=maybe_prefix(prefix, "language_model"),

447 - )

448 - self.register_buffer(

449 - "image_newline_idx_tensor",

450 - torch.tensor([config.image_newline_idx], dtype=torch.long),

451 - persistent=False,

452 - )

453 - self.register_buffer(

454 - "image_new_idx_tensor",

455 - torch.tensor([config.image_new_idx], dtype=torch.long),

456 - persistent=False,

457 - )

427 + with self._mark_tower_model(vllm_config, "image"):

428 + self.vision_tower = init_vision_tower_for_tarsier(

429 + config,

430 + quant_config=quant_config,

431 + multimodal_config=multimodal_config,

432 + require_post_norm=False,

433 + prefix=maybe_prefix(prefix, "vision_tower"),

434 + )

435 + projector_bias = getattr(config, "multimodal_projector_bias", True)

436 +

437 + self.multi_modal_projector = TarsierMultiModalProjector(

438 + vision_hidden_size=config.vision_config.hidden_size,

439 + text_hidden_size=config.text_config.hidden_size,

440 + projector_hidden_act=config.projector_hidden_act,

441 + multimodal_projector_bias=projector_bias,

442 + quant_config=quant_config,

443 + prefix=maybe_prefix(prefix, "multi_modal_projector"),

444 + )

445 + self.register_buffer(

446 + "image_newline_idx_tensor",

447 + torch.tensor([config.image_newline_idx], dtype=torch.long),

448 + persistent=False,

449 + )

450 + self.register_buffer(

451 + "image_new_idx_tensor",

452 + torch.tensor([config.image_new_idx], dtype=torch.long),

453 + persistent=False,

454 + )

455 +

456 + with self._mark_language_model(vllm_config):

457 + self.language_model = init_vllm_registered_model(

458 + vllm_config=vllm_config,

459 + # Use text_config from Tarsier's main config

460 + hf_config=config.text_config,

461 + prefix=maybe_prefix(prefix, "language_model"),

462 + )

463

464 self.make_empty_intermediate_tensors = (

465 self.language_model.make_empty_intermediate_tensors

@@ -547,7 +552,6 @@ class TarsierForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP)

552 self,

553 inputs: TarsierImagePixelInputs,

554 ) -> torch.Tensor | tuple[torch.Tensor, ...]:

550 - assert self.vision_tower is not None

555 pixel_values = inputs["pixel_values"]

556 image_features_selected = self._image_pixels_to_features(

557 self.vision_tower, pixel_values

@@ -575,11 +579,8 @@ class TarsierForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP)

579 "Incorrect type of image_embeds. "

580 f"Got type: {type(projected_features)}. "

581 )

578 - assert self.vision_tower is not None

579 - return self._process_image_pixels(image_input)

582

581 - def get_language_model(self) -> torch.nn.Module:

582 - return self.language_model

583 + return self._process_image_pixels(image_input)

584

585 def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:

586 image_input = self._parse_and_validate_image_input(**kwargs)

vllm/model_executor/models/ultravox.py

+14 -13

@@ -543,7 +543,6 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):

543 assert self.multi_modal_config

544

545 self.secondary_weights = []

546 - self.audio_tower = ModifiedWhisperEncoder(config.audio_config)

546 if config.audio_model_id is not None:

547 # this prefix is not for initialization, but for loading weights

548 # note the trailing dot

@@ -554,15 +553,6 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):

553 prefix="audio_tower.",

554 )

555 )

557 - if config.num_projector_layers > 0:

558 - self.multi_modal_projector = UltravoxTransformerProjector(config)

559 - else:

560 - self.multi_modal_projector = UltravoxFeedForwardProjector(config)

561 - self.language_model = init_vllm_registered_model(

562 - vllm_config=vllm_config,

563 - hf_config=config.wrapped_model_config,

564 - prefix=maybe_prefix(prefix, "language_model"),

565 - )

556 if config.text_model_id is not None:

557 # this prefix is not for initialization, but for loading weights

558 # note the trailing dot

@@ -574,6 +564,20 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):

564 )

565 )

566

567 + with self._mark_tower_model(vllm_config, "audio"):

568 + self.audio_tower = ModifiedWhisperEncoder(config.audio_config)

569 + if config.num_projector_layers > 0:

570 + self.multi_modal_projector = UltravoxTransformerProjector(config)

571 + else:

572 + self.multi_modal_projector = UltravoxFeedForwardProjector(config)

573 +

574 + with self._mark_language_model(vllm_config):

575 + self.language_model = init_vllm_registered_model(

576 + vllm_config=vllm_config,

577 + hf_config=config.wrapped_model_config,

578 + prefix=maybe_prefix(prefix, "language_model"),

579 + )

580 +

581 self.make_empty_intermediate_tensors = (

582 self.language_model.make_empty_intermediate_tensors

583 )

@@ -681,9 +685,6 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):

685 ]

686 return flattened_embeddings.split(embed_lens)

687

684 - def get_language_model(self) -> torch.nn.Module:

685 - return self.language_model

686 -

688 def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:

689 audio_input = self._parse_and_validate_audio_input(**kwargs)

690 if audio_input is None:

vllm/model_executor/models/voxtral.py

+15 -15

@@ -366,22 +366,22 @@ class VoxtralForConditionalGeneration(

366 self.config = config

367 self.downsample_factor = self.config.audio_config.downsample_factor

368

369 - self.language_model = init_vllm_registered_model(

370 - vllm_config=vllm_config,

371 - hf_config=config.text_config,

372 - prefix=maybe_prefix(prefix, "language_model"),

373 - )

374 - self.whisper_encoder = VoxtralEncoderModel(

375 - vllm_config.with_hf_config(config.audio_config),

376 - prefix=maybe_prefix(prefix, "whisper_encoder"),

377 - )

378 - self.audio_language_adapter = AudioLanguageAdapter(

379 - hidden_size=config.audio_config.d_model * self.downsample_factor,

380 - dim=config.text_config.hidden_size,

381 - )

369 + with self._mark_language_model(vllm_config):

370 + self.language_model = init_vllm_registered_model(

371 + vllm_config=vllm_config,

372 + hf_config=config.text_config,

373 + prefix=maybe_prefix(prefix, "language_model"),

374 + )

375

383 - def get_language_model(self) -> torch.nn.Module:

384 - return self.language_model

376 + with self._mark_tower_model(vllm_config, "audio"):

377 + self.whisper_encoder = VoxtralEncoderModel(

378 + vllm_config.with_hf_config(config.audio_config),

379 + prefix=maybe_prefix(prefix, "whisper_encoder"),

380 + )

381 + self.audio_language_adapter = AudioLanguageAdapter(

382 + hidden_size=config.audio_config.d_model * self.downsample_factor,

383 + dim=config.text_config.hidden_size,

384 + )

385

386 def get_mm_mapping(self) -> MultiModelKeys:

387 """Get module prefix for multimodal models to filter LoRA modules."""