@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c46b0cd0af
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 9, 08:45 UTC

[Model][Multimodal] Add explicit MusicFlamingo adapter (#32696)

Signed-off-by: WangHaoyuuu <mailwhaoyu@gmail.com>

Wang Haoyu committed Jan 30, 2026 at 11:01 UTC c46b0cd0af778e5678c52347c5b4957c1e8a0715
6 files changed +115 -2
docs/models/supported_models.md
+1 -1
@@ -657,7 +657,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
657 | Architecture | Models | Inputs | Example HF Models | [LoRA](../features/lora.md) | [PP](../serving/parallelism_scaling.md) |
658 |--------------|--------|--------|-------------------|----------------------|---------------------------|
659 | `AriaForConditionalGeneration` | Aria | T + I<sup>+</sup> | `rhymes-ai/Aria` | | |
660 -| `AudioFlamingo3ForConditionalGeneration` | AudioFlamingo3 | T + A<sup>+</sup> | `nvidia/audio-flamingo-3-hf`, `nvidia/music-flamingo-hf` | ✅︎ | ✅︎ |
660 +| `AudioFlamingo3ForConditionalGeneration` | AudioFlamingo3 | T + A<sup>+</sup> | `nvidia/audio-flamingo-3-hf`, `nvidia/music-flamingo-2601-hf` | ✅︎ | ✅︎ |
661 | `AyaVisionForConditionalGeneration` | Aya Vision | T + I<sup>+</sup> | `CohereLabs/aya-vision-8b`, `CohereLabs/aya-vision-32b`, etc. | | ✅︎ |
662 | `BagelForConditionalGeneration` | BAGEL | T + I<sup>+</sup> | `ByteDance-Seed/BAGEL-7B-MoT` | ✅︎ | ✅︎ |
663 | `BeeForConditionalGeneration` | Bee-8B | T + I<sup>E+</sup> | `Open-Bee/Bee-8B-RL`, `Open-Bee/Bee-8B-SFT` | | ✅︎ |
examples/offline_inference/audio_language.py
+29
@@ -70,6 +70,34 @@ def run_audioflamingo3(question: str, audio_count: int) -> ModelRequestData:
70 )
71
72
73 +# MusicFlamingo
74 +def run_musicflamingo(question: str, audio_count: int) -> ModelRequestData:
75 + model_name = "nvidia/music-flamingo-2601-hf"
76 + engine_args = EngineArgs(
77 + model=model_name,
78 + max_model_len=4096,
79 + max_num_seqs=2,
80 + limit_mm_per_prompt={"audio": audio_count},
81 + enforce_eager=True,
82 + )
83 +
84 + # MusicFlamingo uses <sound> token for audio
85 + audio_placeholder = "<sound>" * audio_count
86 +
87 + prompt = (
88 + "<|im_start|>system\n"
89 + "You are a helpful assistant.<|im_end|>\n"
90 + "<|im_start|>user\n"
91 + f"{audio_placeholder}{question}<|im_end|>\n"
92 + "<|im_start|>assistant\n"
93 + )
94 +
95 + return ModelRequestData(
96 + engine_args=engine_args,
97 + prompt=prompt,
98 + )
99 +
100 +
101 # Gemma3N
102 def run_gemma3n(question: str, audio_count: int) -> ModelRequestData:
103 model_name = "google/gemma-3n-E2B-it"
@@ -452,6 +480,7 @@ def run_whisper(question: str, audio_count: int) -> ModelRequestData:
480
481 model_example_map = {
482 "audioflamingo3": run_audioflamingo3,
483 + "musicflamingo": run_musicflamingo,
484 "gemma3n": run_gemma3n,
485 "glmasr": run_glmasr,
486 "funaudiochat": run_funaudiochat,
tests/models/registry.py
+3
@@ -657,6 +657,9 @@ _MULTIMODAL_EXAMPLE_MODELS = {
657 "AudioFlamingo3ForConditionalGeneration": _HfExamplesInfo(
658 "nvidia/audio-flamingo-3-hf", min_transformers_version="5.0.0"
659 ),
660 + "MusicFlamingoForConditionalGeneration": _HfExamplesInfo(
661 + "nvidia/music-flamingo-2601-hf", min_transformers_version="5.0.0.dev"
662 + ),
663 "AyaVisionForConditionalGeneration": _HfExamplesInfo("CohereLabs/aya-vision-8b"),
664 "BagelForConditionalGeneration": _HfExamplesInfo("ByteDance-Seed/BAGEL-7B-MoT"),
665 "BeeForConditionalGeneration": _HfExamplesInfo(
vllm/model_executor/models/audioflamingo3.py
+8 -1
@@ -128,6 +128,12 @@ class AudioFlamingo3Encoder(Qwen2AudioEncoder):
128 super().__init__(config)
129 self.avg_pooler = nn.AvgPool1d(kernel_size=2, stride=2)
130 # self.layer_norm is already initialized in super().__init__
131 + # Keep a dummy freqs parameter for MusicFlamingo checkpoints.
132 + self.pos_emb = nn.Module()
133 + freqs = torch.empty(getattr(config, "num_mel_bins", 128))
134 + self.pos_emb.register_parameter(
135 + "freqs", nn.Parameter(freqs, requires_grad=False)
136 + )
137
138 def forward(
139 self,
@@ -146,7 +152,8 @@ class AudioFlamingo3Encoder(Qwen2AudioEncoder):
152 ).to(hidden_states.dtype)
153
154 for layer in self.layers:
149 - layer_outputs = layer(hidden_states, attention_mask)
155 + # Qwen2AudioEncoderLayer expects layer_head_mask as third arg.
156 + layer_outputs = layer(hidden_states, attention_mask, None)
157 hidden_states = layer_outputs[0]
158
159 # AvgPool (time/2) + LayerNorm
vllm/model_executor/models/musicflamingo.py new
+70
@@ -0,0 +1,70 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +"""MusicFlamingo model adapter.
5 +
6 +MusicFlamingo shares the AudioFlamingo3 architecture, so we reuse the same
7 +implementation and multimodal processor, while accepting MusicFlamingo config
8 +and processor classes when available.
9 +"""
10 +
11 +from collections.abc import Mapping
12 +
13 +from transformers.models.audioflamingo3 import (
14 + AudioFlamingo3Config,
15 + AudioFlamingo3Processor,
16 +)
17 +
18 +from vllm.multimodal import MULTIMODAL_REGISTRY
19 +from vllm.multimodal.processing import BaseProcessingInfo
20 +
21 +from .audioflamingo3 import (
22 + AudioFlamingo3DummyInputsBuilder,
23 + AudioFlamingo3ForConditionalGeneration,
24 + AudioFlamingo3MultiModalProcessor,
25 +)
26 +
27 +try:
28 + # Optional dependency: use MusicFlamingo classes when transformers provides them.
29 + from transformers.models.musicflamingo import (
30 + MusicFlamingoConfig,
31 + MusicFlamingoProcessor,
32 + )
33 +except Exception: # pragma: no cover - optional dependency
34 + MusicFlamingoConfig = None
35 + MusicFlamingoProcessor = None
36 +
37 +
38 +class MusicFlamingoProcessingInfo(BaseProcessingInfo):
39 + def get_hf_config(self):
40 + if MusicFlamingoConfig is None:
41 + return self.ctx.get_hf_config(AudioFlamingo3Config)
42 + return self.ctx.get_hf_config((MusicFlamingoConfig, AudioFlamingo3Config))
43 +
44 + def get_hf_processor(self, **kwargs: object):
45 + if MusicFlamingoProcessor is None:
46 + return self.ctx.get_hf_processor(AudioFlamingo3Processor, **kwargs)
47 + # Tuple triggers AutoProcessor path and accepts either processor class.
48 + return self.ctx.get_hf_processor(
49 + (MusicFlamingoProcessor, AudioFlamingo3Processor), **kwargs
50 + )
51 +
52 + def get_feature_extractor(self, **kwargs: object):
53 + hf_processor = self.get_hf_processor(**kwargs)
54 + return hf_processor.feature_extractor
55 +
56 + def get_supported_mm_limits(self) -> Mapping[str, int | None]:
57 + return {"audio": None}
58 +
59 +
60 +class MusicFlamingoDummyInputsBuilder(AudioFlamingo3DummyInputsBuilder):
61 + pass
62 +
63 +
64 +@MULTIMODAL_REGISTRY.register_processor(
65 + AudioFlamingo3MultiModalProcessor,
66 + info=MusicFlamingoProcessingInfo,
67 + dummy_inputs=MusicFlamingoDummyInputsBuilder,
68 +)
69 +class MusicFlamingoForConditionalGeneration(AudioFlamingo3ForConditionalGeneration):
70 + """MusicFlamingo model for conditional generation."""
vllm/model_executor/models/registry.py
+4
@@ -286,6 +286,10 @@ _MULTIMODAL_MODELS = {
286 "audioflamingo3",
287 "AudioFlamingo3ForConditionalGeneration",
288 ),
289 + "MusicFlamingoForConditionalGeneration": (
290 + "musicflamingo",
291 + "MusicFlamingoForConditionalGeneration",
292 + ),
293 "AyaVisionForConditionalGeneration": (
294 "aya_vision",
295 "AyaVisionForConditionalGeneration",