@tduncan / Nvidia-Optimization-Stack-VLLM / commits / a22e0dfc69

[Model] Remove AyaVision, MusicFlamingo (#47263)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>

Harry Mellor committed Jul 1, 2026 at 11:39 UTC a22e0dfc69a0fe9e058ccea865fbc4868bb187e1

12 files changed +3 -1422

docs/models/supported_models.md

-2

@@ -532,7 +532,6 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen

532 | ------------ | ------ | ------ | ----------------- | -------------------- | ------------------------- |

534 | `AudioFlamingo3ForConditionalGeneration` | AudioFlamingo3 | T + A | `nvidia/audio-flamingo-3-hf`, `nvidia/music-flamingo-hf` | ✅︎ | ✅︎ |

535 -| `AyaVisionForConditionalGeneration` | Aya Vision | T + I+ | `CohereLabs/aya-vision-8b`, `CohereLabs/aya-vision-32b`, etc. | | ✅︎ |

535 | `BagelForConditionalGeneration` | BAGEL | T + I+ | `ByteDance-Seed/BAGEL-7B-MoT` | ✅︎ | ✅︎ |

536 | `BeeForConditionalGeneration` | Bee-8B | T + IE+ | `Open-Bee/Bee-8B-RL`, `Open-Bee/Bee-8B-SFT` | | ✅︎ |

537 | `Blip2ForConditionalGeneration` | BLIP-2 | T + IE | `Salesforce/blip2-opt-2.7b`, `Salesforce/blip2-opt-6.7b`, etc. | ✅︎ | ✅︎ |

@@ -594,7 +593,6 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen

593 | `Molmo2ForConditionalGeneration` | Molmo2 | T + I+ / V | `allenai/Molmo2-4B`, `allenai/Molmo2-8B`, `allenai/Molmo2-O-7B`, `allenai/MolmoWeb-4B`^, `allenai/MolmoWeb-8B`^ | ✅︎ | ✅︎ |

594 | `MossAudioModel` | MOSS-Audio | T + A+ | `OpenMOSS-Team/MOSS-Audio-4B-Instruct`, `OpenMOSS-Team/MOSS-Audio-4B-Thinking`, `OpenMOSS-Team/MOSS-Audio-8B-Instruct`, `OpenMOSS-Team/MOSS-Audio-8B-Thinking` | ✅︎ | ✅︎ |

595 | `Moondream3ForCausalLM` | Moondream3 | T + I | `moondream/moondream3-preview` | | ✅︎ |

597 -| `MusicFlamingoForConditionalGeneration` | MusicFlamingo | T + A | `nvidia/music-flamingo-2601-hf`, `nvidia/music-flamingo-think-2601-hf` | ✅︎ | ✅︎ |

596 | `NVLM_D_Model` | NVLM-D 1.0 | T + I+ | `nvidia/NVLM-D-72B`, etc. | | ✅︎ |

597 | `OpenCUAForConditionalGeneration` | OpenCUA-7B | T + IE+ | `xlangai/OpenCUA-7B` | ✅︎ | ✅︎ |

598 | `OpenPanguVLForConditionalGeneration` | openpangu-VL | T + IE+ + VE+ | `FreedomIntelligence/openPangu-VL-7B` | ✅︎ | ✅︎ |

examples/generate/multimodal/audio_language_offline.py

-39

@@ -91,44 +91,6 @@ def run_cohere_asr(question: str, audio_count: int) -> ModelRequestData:

91 )

94 -# MusicFlamingo

95 -def run_musicflamingo(question: str, audio_count: int) -> ModelRequestData:

96 - model_name = "nvidia/music-flamingo-2601-hf"

97 - engine_args = EngineArgs(

98 - model=model_name,

99 - max_model_len=4096,

100 - max_num_seqs=2,

101 - limit_mm_per_prompt={"audio": audio_count},

102 - enforce_eager=True,

103 - )

104 -

105 - # MusicFlamingo prompt placeholders use <sound>; vLLM's MusicFlamingo

106 - # multimodal processor expands each one into <|sound_bos|> + audio tokens +

107 - # <|sound_eos|> based on extracted audio feature lengths.

108 - audio_placeholder = "<sound>" * audio_count

109 - system_prompt = (

110 - "You are Music Flamingo, a multimodal assistant for language and music. "

111 - "On each turn you receive an audio clip which contains music and optional "

112 - "text, you will receive at least one or both; use your world knowledge and "

113 - "reasoning to help the user with any task. Interpret the entirety of the "

114 - "content any input music--regardlenss of whether the user calls it audio, "

115 - "music, or sound."

116 - )

117 -

118 - prompt = (

119 - "<|im_start|>system\n"

120 - f"{system_prompt}<|im_end|>\n"

121 - "<|im_start|>user\n"

122 - f"{audio_placeholder}{question}<|im_end|>\n"

123 - "<|im_start|>assistant\n"

124 - )

125 -

126 - return ModelRequestData(

127 - engine_args=engine_args,

128 - prompt=prompt,

129 - )

130 -

131 -

94 # Gemma3N

95 def run_gemma3n(question: str, audio_count: int) -> ModelRequestData:

96 model_name = "google/gemma-3n-E2B-it"

@@ -565,7 +527,6 @@ model_example_map = {

527 "kimi_audio": run_kimi_audio,

528 "midashenglm": run_midashenglm,

529 "minicpmo": run_minicpmo,

568 - "musicflamingo": run_musicflamingo,

530 "phi4_mm": run_phi4mm,

531 "qwen2_audio": run_qwen2_audio,

532 "qwen2_5_omni": run_qwen2_5_omni,

examples/generate/multimodal/vision_language_multi_image_offline.py

-34

@@ -74,39 +74,6 @@ def load_aria(question: str, image_urls: list[str]) -> ModelRequestData:

74 )

77 -def load_aya_vision(question: str, image_urls: list[str]) -> ModelRequestData:

78 - model_name = "CohereLabs/aya-vision-8b"

79 -

80 - engine_args = EngineArgs(

81 - model=model_name,

82 - max_num_seqs=2,

83 - limit_mm_per_prompt={"image": len(image_urls)},

84 - )

85 -

86 - placeholders = [{"type": "image", "image": url} for url in image_urls]

87 - messages = [

88 - {

89 - "role": "user",

90 - "content": [

91 - *placeholders,

92 - {"type": "text", "text": question},

93 - ],

94 - }

95 - ]

96 -

97 - processor = AutoProcessor.from_pretrained(model_name)

98 -

99 - prompt = processor.apply_chat_template(

100 - messages, tokenize=False, add_generation_prompt=True

101 - )

102 -

103 - return ModelRequestData(

104 - engine_args=engine_args,

105 - prompt=prompt,

106 - image_data=[fetch_image(url) for url in image_urls],

107 - )

108 -

109 -

77 def load_bee(question: str, image_urls: list[str]) -> ModelRequestData:

78 model_name = "Open-Bee/Bee-8B-RL"

@@ -1420,7 +1387,6 @@ def load_molmo2(question: str, image_urls: list[str]) -> ModelRequestData:

1387

1388 model_example_map = {

1389 "aria": load_aria,

1423 - "aya_vision": load_aya_vision,

1390 "bee": load_bee,

1391 "command_a_vision": load_command_a_vision,

1392 "deepseek_vl_v2": load_deepseek_vl2,

examples/generate/multimodal/vision_language_offline.py

-23

@@ -68,28 +68,6 @@ def run_aria(questions: list[str], modality: str) -> ModelRequestData:

68 )

71 -# Aya Vision

72 -def run_aya_vision(questions: list[str], modality: str) -> ModelRequestData:

73 - assert modality == "image"

74 - model_name = "CohereLabs/aya-vision-8b"

75 -

76 - engine_args = EngineArgs(

77 - model=model_name,

78 - max_model_len=2048,

79 - max_num_seqs=2,

80 - mm_processor_kwargs={"crop_to_patches": True},

81 - limit_mm_per_prompt={modality: 1},

82 - )

83 - prompts = [

84 - f"<|START_OF_TURN_TOKEN|><|USER_TOKEN|><image>{question}<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>"

85 - for question in questions

86 - ]

87 - return ModelRequestData(

88 - engine_args=engine_args,

89 - prompts=prompts,

90 - )

91 -

92 -

71 # Bee-8B

72 def run_bee(questions: list[str], modality: str) -> ModelRequestData:

73 assert modality == "image"

@@ -2327,7 +2305,6 @@ def run_step_vl(questions: list[str], modality: str) -> ModelRequestData:

2305

2306 model_example_map = {

2307 "aria": run_aria,

2330 - "aya_vision": run_aya_vision,

2308 "bagel": run_bagel,

2309 "cheers": run_cheers,

2310 "bee": run_bee,

tests/models/multimodal/generation/test_common.py

-33

@@ -326,39 +326,6 @@ VLM_TEST_SETTINGS = {

326 large_gpu_mark(min_gb=64),

327 ],

328 ),

329 - "aya_vision": VLMTestInfo(

330 - models=["CohereLabs/aya-vision-8b"],

331 - test_type=(VLMTestType.IMAGE),

332 - prompt_formatter=lambda img_prompt: f"<|START_OF_TURN_TOKEN|><|USER_TOKEN|>{img_prompt}<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>", # noqa: E501

333 - single_image_prompts=IMAGE_ASSETS.prompts(

334 - {

335 - "stop_sign": "<image>What's the content in the center of the image?",

336 - "cherry_blossom": "<image>What is the season?",

337 - }

338 - ),

339 - multi_image_prompt="<image><image>Describe the two images in detail.",

340 - max_model_len=4096,

341 - max_num_seqs=2,

342 - auto_cls=AutoModelForImageTextToText,

343 - vllm_runner_kwargs={"mm_processor_kwargs": {"crop_to_patches": True}},

344 - ),

345 - "aya_vision-multi_image": VLMTestInfo(

346 - models=["CohereLabs/aya-vision-8b"],

347 - test_type=(VLMTestType.MULTI_IMAGE),

348 - prompt_formatter=lambda img_prompt: f"<|START_OF_TURN_TOKEN|><|USER_TOKEN|>{img_prompt}<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>", # noqa: E501

349 - single_image_prompts=IMAGE_ASSETS.prompts(

350 - {

351 - "stop_sign": "<image>What's the content in the center of the image?",

352 - "cherry_blossom": "<image>What is the season?",

353 - }

354 - ),

355 - multi_image_prompt="<image><image>Describe the two images in detail.",

356 - max_model_len=4096,

357 - max_num_seqs=2,

358 - auto_cls=AutoModelForImageTextToText,

359 - vllm_runner_kwargs={"mm_processor_kwargs": {"crop_to_patches": True}},

360 - marks=[large_gpu_mark(min_gb=32)],

361 - ),

329 "blip2": VLMTestInfo(

330 models=["Salesforce/blip2-opt-2.7b"],

331 test_type=VLMTestType.IMAGE,

tests/models/multimodal/generation/test_musicflamingo.py deleted

-152

@@ -1,152 +0,0 @@

1 -# SPDX-License-Identifier: Apache-2.0

2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -

4 -import json

5 -import os

6 -

7 -import pytest

8 -

9 -from tests.models.registry import HF_EXAMPLE_MODELS

10 -from vllm import LLM, SamplingParams

11 -

12 -MODEL_NAME = "nvidia/music-flamingo-2601-hf"

13 -SINGLE_CONVERSATION = [

14 - {

15 - "role": "user",

16 - "content": [

17 - {

18 - "type": "text",

19 - "text": "Describe this track in full detail - tell me the "

20 - "genre, tempo, and key, then dive into the instruments, "

21 - "production style, and overall mood it creates.",

22 - },

23 - {

24 - "type": "audio_url",

25 - "audio_url": {

26 - "url": "https://huggingface.co/datasets/nvidia/AudioSkills/"

27 - "resolve/main/assets/song_1.mp3",

28 - },

29 - },

30 - ],

31 - }

32 -]

33 -BATCHED_CONVERSATIONS = [

34 - SINGLE_CONVERSATION,

35 - [

36 - {

37 - "role": "user",

38 - "content": [

39 - {

40 - "type": "text",

41 - "text": "Generate a structured lyric sheet from the input music.",

42 - },

43 - {

44 - "type": "audio_url",

45 - "audio_url": {

46 - "url": "https://huggingface.co/datasets/nvidia/"

47 - "AudioSkills/resolve/main/assets/song_2.mp3",

48 - },

49 - },

50 - ],

51 - }

52 - ],

53 -]

54 -

55 -

56 -def get_fixture_path(filename):

57 - return os.path.join(

58 - os.path.dirname(__file__), "../../fixtures/musicflamingo", filename

59 - )

60 -

61 -

62 -def load_expected_fixture(filename):

63 - fixture_path = get_fixture_path(filename)

64 - with open(fixture_path) as f:

65 - return json.load(f)

66 -

67 -

68 -def assert_output_matches(output, expected_text, expected_token_ids):

69 - generated = output.outputs[0]

70 - assert generated.text == expected_text

71 - actual_token_ids = list(generated.token_ids)

72 - assert (

73 - actual_token_ids == expected_token_ids

74 - or actual_token_ids == expected_token_ids[:-1]

75 - or actual_token_ids[:-1] == expected_token_ids

76 - )

77 -

78 -

79 -@pytest.fixture(scope="module")

80 -def llm():

81 - model_info = HF_EXAMPLE_MODELS.get_hf_info("MusicFlamingoForConditionalGeneration")

82 - model_info.check_transformers_version(on_fail="skip")

83 -

84 - try:

85 - llm = LLM(

86 - model=MODEL_NAME,

87 - dtype="bfloat16",

88 - enforce_eager=True,

89 - max_model_len=8192,

90 - limit_mm_per_prompt={"audio": 1},

91 - )

92 - except Exception as e:

93 - pytest.skip(f"Failed to load model {MODEL_NAME}: {e}")

94 -

95 - # ROCm may compile decoder kernels on the first inference pass; warm up

96 - # once so exact fixture assertions cover the steady-state path.

97 - llm.chat(

98 - messages=SINGLE_CONVERSATION,

99 - sampling_params=SamplingParams(temperature=0.0, max_tokens=1),

100 - use_tqdm=False,

101 - )

102 -

103 - return llm

104 -

105 -

106 -def test_single_generation(llm):

107 - expected = load_expected_fixture("expected_results_single.json")

108 -

109 - outputs = llm.chat(

110 - messages=SINGLE_CONVERSATION,

111 - sampling_params=SamplingParams(temperature=0.0, max_tokens=50),

112 - )

113 -

114 - assert_output_matches(

115 - outputs[0],

116 - expected["transcriptions"][0],

117 - expected["token_ids"][0],

118 - )

119 -

120 -

121 -def test_batched_generation(llm):

122 - expected = load_expected_fixture("expected_results_batched.json")

123 -

124 - outputs = llm.chat(

125 - messages=BATCHED_CONVERSATIONS,

126 - sampling_params=SamplingParams(temperature=0.0, max_tokens=50),

127 - )

128 -

129 - for i, output in enumerate(outputs):

130 - assert_output_matches(

131 - output,

132 - expected["transcriptions"][i],

133 - expected["token_ids"][i],

134 - )

135 -

136 -

137 -def test_single_and_batched_generation_match(llm):

138 - sampling_params = SamplingParams(temperature=0.0, max_tokens=50)

139 -

140 - single_output = llm.chat(

141 - messages=SINGLE_CONVERSATION,

142 - sampling_params=sampling_params,

143 - )[0]

144 - batched_output = llm.chat(

145 - messages=BATCHED_CONVERSATIONS,

146 - sampling_params=sampling_params,

147 - )[0]

148 -

149 - assert single_output.outputs[0].text == batched_output.outputs[0].text

150 - assert list(single_output.outputs[0].token_ids) == list(

151 - batched_output.outputs[0].token_ids

152 - )

tests/models/multimodal/processing/test_musicflamingo.py deleted

-228

@@ -1,228 +0,0 @@

1 -# SPDX-License-Identifier: Apache-2.0

2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -

6 -# reserved.

7 -#

8 -# Licensed under the Apache License, Version 2.0 (the "License");

9 -# you may not use this file except in compliance with the License.

10 -# You may obtain a copy of the License at

11 -#

12 -# http://www.apache.org/licenses/LICENSE-2.0

13 -#

14 -# Unless required by applicable law or agreed to in writing, software

15 -# distributed under the License is distributed on an "AS IS" BASIS,

16 -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

17 -# See the License for the specific language governing permissions and

18 -# limitations under the License.

19 -

20 -from importlib.metadata import version

21 -from unittest.mock import MagicMock

22 -

23 -import numpy as np

24 -import pytest

25 -import torch

26 -from packaging.version import Version

27 -from transformers import PretrainedConfig

28 -

29 -from tests.models.registry import HF_EXAMPLE_MODELS

30 -

31 -

32 -class MockMusicFlamingoConfig(PretrainedConfig):

33 - model_type = "musicflamingo"

34 -

35 - def __init__(self, **kwargs):

36 - super().__init__(**kwargs)

37 - self.audio_config = PretrainedConfig()

38 - self.text_config = PretrainedConfig()

39 -

40 -

41 -class MockMusicFlamingoProcessor:

42 - def __init__(self):

43 - self.audio_token = "<sound>"

44 - self.audio_token_id = 12345

45 - self.audio_bos_token = "<|sound_bos|>"

46 - self.audio_bos_token_id = 12346

47 - self.audio_eos_token = "<|sound_eos|>"

48 - self.audio_eos_token_id = 12347

49 - self.max_audio_len = 1200

50 - self.feature_extractor = MockFeatureExtractor()

51 -

52 - def __call__(self, text=None, audio=None, **kwargs):

53 - return {

54 - "input_ids": torch.tensor([[1, 2, 3]], dtype=torch.long),

55 - "input_features": torch.zeros((3, 80, 3000)),

56 - "input_features_mask": torch.ones((3, 3000), dtype=torch.long),

57 - }

58 -

59 -

60 -class MockFeatureExtractor:

61 - def __init__(self):

62 - self.sampling_rate = 16000

63 - self.chunk_length = 30

64 -

65 -

66 -@pytest.fixture

67 -def mock_ctx():

68 - config = MockMusicFlamingoConfig()

69 -

70 - ctx = MagicMock()

71 - ctx.get_hf_config.return_value = config

72 - ctx.get_hf_processor.return_value = MockMusicFlamingoProcessor()

73 - ctx.call_hf_processor.side_effect = lambda processor, data, kwargs: processor(

74 - **data, **kwargs

75 - )

76 - ctx.model_config.hf_config = config

77 - return ctx

78 -

79 -

80 -@pytest.fixture(autouse=True)

81 -def check_transformers_version():

82 - model_info = HF_EXAMPLE_MODELS.get_hf_info("MusicFlamingoForConditionalGeneration")

83 - model_info.check_transformers_version(on_fail="skip")

84 -

85 -

86 -def test_musicflamingo_chunk_counting_without_rote_timestamps(mock_ctx):

87 - from vllm.model_executor.models.musicflamingo import (

88 - MusicFlamingoDummyInputsBuilder,

89 - MusicFlamingoMultiModalProcessor,

90 - MusicFlamingoProcessingInfo,

91 - )

92 -

93 - info = MusicFlamingoProcessingInfo(mock_ctx)

94 - processor = MusicFlamingoMultiModalProcessor(

95 - info, MusicFlamingoDummyInputsBuilder(info)

96 - )

97 -

98 - sr = 16000

99 - audio_1 = np.zeros(30 * sr)

100 - audio_2 = np.zeros(45 * sr)

101 -

102 - mm_data = {"audio": [audio_1, audio_2]}

103 - prompt = "<|user|>Listen.<|end|>"

104 -

105 - processed = processor._call_hf_processor(prompt, mm_data, {}, {})

106 -

107 - chunk_counts = processed["chunk_counts"]

108 -

109 - assert chunk_counts.tolist() == [1, 2]

110 - assert "rote_timestamps" not in processed

111 - assert processed["feature_attention_mask"].shape == (3, 3000)

112 -

113 -

114 -def test_musicflamingo_dummy_text_uses_plain_audio_tokens(mock_ctx):

115 - from vllm.model_executor.models.musicflamingo import (

116 - MusicFlamingoDummyInputsBuilder,

117 - MusicFlamingoProcessingInfo,

118 - )

119 -

120 - info = MusicFlamingoProcessingInfo(mock_ctx)

121 - builder = MusicFlamingoDummyInputsBuilder(info)

122 -

123 - assert builder.get_dummy_text({"audio": 2}) == "<sound><sound>"

124 -

125 -

126 -@pytest.mark.skipif(

127 - Version(version("transformers")) >= Version("5.5"),

128 - reason="transformers v5.5 added native MusicFlamingoForConditionalGeneration "

129 - "with a different get_audio_features signature (requires input_ids)",

130 -)

131 -def test_musicflamingo_audio_feature_pipeline_matches_hf_small_config():

132 - from transformers.models.musicflamingo import (

133 - modeling_musicflamingo as hf_musicflamingo_modeling,

134 - )

135 - from transformers.models.musicflamingo.configuration_musicflamingo import (

136 - MusicFlamingoConfig,

137 - )

138 -

139 - from vllm.model_executor.models.audioflamingo3 import (

140 - _build_audio_encoder_attention_mask,

141 - _flatten_valid_audio_embeddings,

142 - )

143 - from vllm.model_executor.models.musicflamingo import (

144 - MusicFlamingoEncoder,

145 - MusicFlamingoMultiModalProjector,

146 - MusicFlamingoRotaryEmbedding,

147 - apply_rotary_time_emb,

148 - )

149 -

150 - text_config = {

151 - "model_type": "qwen2",

152 - "intermediate_size": 64,

153 - "initializer_range": 0.02,

154 - "hidden_size": 32,

155 - "max_position_embeddings": 1024,

156 - "num_hidden_layers": 2,

157 - "num_attention_heads": 4,

158 - "num_key_value_heads": 2,

159 - "vocab_size": 128,

160 - "pad_token_id": 1,

161 - "use_mrope": False,

162 - }

163 - audio_config = {

164 - "hidden_size": 16,

165 - "num_attention_heads": 4,

166 - "intermediate_size": 32,

167 - "num_hidden_layers": 2,

168 - "num_mel_bins": 80,

169 - "max_source_positions": 1500,

170 - "dropout": 0.0,

171 - "attention_dropout": 0.0,

172 - "activation_dropout": 0.0,

173 - "encoder_layerdrop": 0.0,

174 - }

175 -

176 - torch.manual_seed(0)

177 - config = MusicFlamingoConfig(

178 - text_config=text_config,

179 - audio_config=audio_config,

180 - audio_token_id=0,

181 - head_dim=8,

182 - rope_parameters={"rope_type": "default", "rope_theta": 2048},

183 - )

184 - hf_model = hf_musicflamingo_modeling.MusicFlamingoForConditionalGeneration(

185 - config

186 - ).eval()

187 -

188 - vllm_encoder = MusicFlamingoEncoder(config.audio_config).eval()

189 - vllm_encoder.load_state_dict(hf_model.audio_tower.state_dict())

190 -

191 - vllm_projector = MusicFlamingoMultiModalProjector(config).eval()

192 - vllm_projector.load_state_dict(hf_model.multi_modal_projector.state_dict())

193 -

194 - vllm_rope = MusicFlamingoRotaryEmbedding(config).eval()

195 - vllm_rope.load_state_dict(hf_model.pos_emb.state_dict(), strict=False)

196 -

197 - input_features = torch.randn(3, 80, 3000)

198 - feature_attention_mask = torch.zeros(3, 3000, dtype=torch.bool)

199 - feature_attention_mask[0, :3000] = True

200 - feature_attention_mask[1, :2500] = True

201 - feature_attention_mask[2, :1500] = True

202 - rote_timestamps = (

203 - torch.arange(750, dtype=torch.float32).unsqueeze(0).repeat(3, 1) * 0.04

204 - )

205 -

206 - hf_output = hf_model.get_audio_features(

207 - input_features,

208 - feature_attention_mask,

209 - rote_timestamps=rote_timestamps,

210 - return_dict=True,

211 - ).pooler_output

212 - vllm_attention_mask = _build_audio_encoder_attention_mask(

213 - feature_attention_mask,

214 - dtype=vllm_encoder.conv1.weight.dtype,

215 - device=vllm_encoder.conv1.weight.device,

216 - )

217 - vllm_hidden_states = vllm_encoder(

218 - input_features,

219 - attention_mask=vllm_attention_mask,

220 - )

221 - cos, sin = vllm_rope(rote_timestamps, seq_len=vllm_hidden_states.shape[-2])

222 - vllm_hidden_states = apply_rotary_time_emb(vllm_hidden_states, cos, sin)

223 - vllm_output, _ = _flatten_valid_audio_embeddings(

224 - vllm_projector(vllm_hidden_states),

225 - feature_attention_mask,

226 - )

227 -

228 - torch.testing.assert_close(vllm_output, hf_output)

tests/models/registry.py

-8

@@ -780,14 +780,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {

780 "vllm": "Needs https://github.com/huggingface/transformers/pull/43538"

781 },

782 ),

783 - "MusicFlamingoForConditionalGeneration": _HfExamplesInfo(

784 - "nvidia/music-flamingo-2601-hf",

785 - min_transformers_version="5.5.0",

786 - transformers_version_reason={

787 - "vllm": "Needs https://github.com/huggingface/transformers/pull/43538"

788 - },

789 - ),

790 - "AyaVisionForConditionalGeneration": _HfExamplesInfo("CohereLabs/aya-vision-8b"),

783 "BagelForConditionalGeneration": _HfExamplesInfo("ByteDance-Seed/BAGEL-7B-MoT"),

784 "BeeForConditionalGeneration": _HfExamplesInfo(

785 "Open-Bee/Bee-8B-RL",

vllm/model_executor/models/aya_vision.py deleted

-445

@@ -1,445 +0,0 @@

1 -# SPDX-License-Identifier: Apache-2.0

2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -# Adapted from https://github.com/huggingface/transformers/tree/main/src/transformers/models/aya_vision

4 -from collections.abc import Iterable, Mapping, Sequence

5 -from typing import Annotated, Literal

6 -

7 -import torch

8 -from torch import nn

9 -from transformers import BatchFeature, GotOcr2ImageProcessor

10 -from transformers.activations import ACT2FN

11 -from transformers.image_processing_utils import get_size_dict

12 -from transformers.models.aya_vision import AyaVisionConfig

13 -from transformers.models.aya_vision.processing_aya_vision import AyaVisionProcessor

14 -from transformers.models.got_ocr2.image_processing_got_ocr2 import (

15 - get_optimal_tiled_canvas,

16 -)

17 -

18 -from vllm.config import VllmConfig

19 -from vllm.config.multimodal import BaseDummyOptions

20 -from vllm.inputs import MultiModalDataDict

21 -from vllm.multimodal import MULTIMODAL_REGISTRY

22 -from vllm.multimodal.inputs import (

23 - MultiModalFieldConfig,

24 - MultiModalKwargsItems,

25 -)

26 -from vllm.multimodal.parse import ImageProcessorItems, ImageSize, MultiModalDataItems

27 -from vllm.multimodal.processing import (

28 - BaseDummyInputsBuilder,

29 - BaseMultiModalProcessor,

30 - BaseProcessingInfo,

31 - PromptReplacement,

32 - PromptUpdate,

33 - PromptUpdateDetails,

34 -)

35 -from vllm.sequence import IntermediateTensors

36 -from vllm.utils.tensor_schema import TensorSchema, TensorShape

37 -

38 -from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP

39 -from .siglip import SiglipVisionModel

40 -from .utils import (

41 - AutoWeightsLoader,

42 - WeightsMapper,

43 - get_layer_index,

44 - init_vllm_registered_model,

45 - maybe_prefix,

46 -)

47 -

48 -

49 -class AyaVisionImagePixelInputs(TensorSchema):

50 - """

51 - Dimensions:

52 - - np: The total number of patches over each image over each prompt in

53 - the batch

54 - - c: Number of channels

55 - - h: Height of each image patch

56 - - w: Width of each image patch

57 - - bn: Batch size * number of images

58 - """

59 -

60 - type: Literal["pixel_values"]

61 -

62 - pixel_values: Annotated[

63 - torch.Tensor,

64 - TensorShape("np", 3, "h", "w"),

65 - ]

66 -

67 - num_patches: Annotated[

68 - torch.Tensor,

69 - TensorShape("bn"),

70 - ]

71 -

72 -

73 -class AyaVisionMultiModalProjector(nn.Module):

74 - def __init__(self, config: AyaVisionConfig):

75 - super().__init__()

76 - self.config = config

77 - self.downsample_factor = config.downsample_factor

78 - self.alignment_intermediate_size = getattr(

79 - config, "alignment_intermediate_size", config.text_config.hidden_size

80 - )

81 - self.layernorm = nn.LayerNorm(

82 - config.vision_config.hidden_size * (config.downsample_factor**2),

83 - eps=config.adapter_layer_norm_eps,

84 - )

85 -

86 - self.linear_1 = nn.Linear(

87 - config.vision_config.hidden_size * (config.downsample_factor**2),

88 - self.alignment_intermediate_size,

89 - bias=True,

90 - )

91 -

92 - self.act = ACT2FN["silu"] # SwiGLU uses SiLU activation

93 - # For SwiGLU, project down to half size since we split intermediate dim

94 - self.linear_2 = nn.Linear(

95 - self.alignment_intermediate_size // 2,

96 - config.text_config.hidden_size,

97 - bias=True,

98 - )

99 -

100 - def forward(self, image_features: torch.Tensor) -> torch.Tensor:

101 - image_features = self.pixel_shuffle(image_features)

102 - image_features = self.layernorm(image_features)

103 - hidden_states = self.linear_1(image_features)

104 -

105 - # Split along last dimension and apply SwiGLU

106 - x, gate = hidden_states.chunk(2, dim=-1)

107 - hidden_states = self.act(gate) * x

108 -

109 - hidden_states = self.linear_2(hidden_states)

110 - return hidden_states

111 -

112 - def pixel_shuffle(self, image_features: torch.Tensor) -> torch.Tensor: # B, S, D

113 - batch_size, seq_length, _ = image_features.shape

114 - height = width = int(seq_length**0.5)

115 - image_features = image_features.reshape(

116 - image_features.shape[0], width, height, -1

117 - )

118 - channels = image_features.shape[-1]

119 - image_features = image_features.reshape(

120 - batch_size,

121 - width,

122 - int(height / self.downsample_factor),

123 - int(channels * self.downsample_factor),

124 - )

125 - image_features = image_features.permute(0, 2, 1, 3)

126 - image_features = image_features.reshape(

127 - batch_size,

128 - int(height / self.downsample_factor),

129 - int(width / self.downsample_factor),

130 - -1,

131 - )

132 - image_features = image_features.permute(0, 2, 1, 3)

133 - return image_features

134 -

135 -

136 -class AyaVisionProcessingInfo(BaseProcessingInfo):

137 - def get_hf_config(self) -> AyaVisionConfig:

138 - return self.ctx.get_hf_config(AyaVisionConfig)

139 -

140 - def get_hf_processor(self, **kwargs: object) -> AyaVisionProcessor:

141 - return self.ctx.get_hf_processor(AyaVisionProcessor, **kwargs)

142 -

143 - def get_image_processor(self, **kwargs: object) -> GotOcr2ImageProcessor:

144 - return self.get_hf_processor(**kwargs).image_processor

145 -

146 - def get_supported_mm_limits(self) -> Mapping[str, int | None]:

147 - return {"image": None}

148 -

149 - def get_image_size_with_most_features(self) -> ImageSize:

150 - image_processor = self.get_image_processor()

151 - height = image_processor.size["height"]

152 - width = image_processor.size["width"]

153 - max_patches = image_processor.max_patches

154 - return ImageSize(height=height * max_patches, width=width * max_patches)

155 -

156 - def get_num_patches(

157 - self,

158 - *,

159 - image_width: int,

160 - image_height: int,

161 - size: dict,

162 - min_patches: int,

163 - max_patches: int,

164 - ) -> int:

165 - """

166 - Calculate the number of patches needed for a given image based on size

167 - constraints. This method replicates and adjusts the logic from:

168 - transformers/models/got_ocr2/image_processing_got_ocr2

169 - """

170 - size = get_size_dict(size, default_to_square=False)

171 - num_columns, num_rows = get_optimal_tiled_canvas(

172 - (image_height, image_width),

173 - (size["height"], size["width"]),

174 - min_patches,

175 - max_patches,

176 - )

177 - num_blocks = num_columns * num_rows

178 - return num_blocks if num_blocks == 1 else num_blocks + 1

179 -

180 -

181 -class AyaVisionDummyInputsBuilder(BaseDummyInputsBuilder[AyaVisionProcessingInfo]):

182 - def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:

183 - num_images = mm_counts.get("image", 0)

184 -

185 - processor = self.info.get_hf_processor()

186 - image_token = processor.image_token

187 -

188 - return image_token * num_images

189 -

190 - def get_dummy_mm_data(

191 - self,

192 - seq_len: int,

193 - mm_counts: Mapping[str, int],

194 - mm_options: Mapping[str, BaseDummyOptions],

195 - ) -> MultiModalDataDict:

196 - num_images = mm_counts.get("image", 0)

197 - image_size = self.info.get_image_size_with_most_features()

198 -

199 - image_overrides = mm_options.get("image")

200 -

201 - return {

202 - "image": self._get_dummy_images(

203 - width=image_size.width,

204 - height=image_size.height,

205 - num_images=num_images,

206 - overrides=image_overrides,

207 - )

208 - }

209 -

210 -

211 -class AyaVisionMultiModalProcessor(BaseMultiModalProcessor[AyaVisionProcessingInfo]):

212 - def _call_hf_processor(

213 - self,

214 - prompt: str,

215 - mm_data: Mapping[str, object],

216 - mm_kwargs: Mapping[str, object],

217 - tok_kwargs: Mapping[str, object],

218 - ) -> BatchFeature:

219 - processed_outputs = super()._call_hf_processor(

220 - prompt,

221 - mm_data,

222 - mm_kwargs,

223 - tok_kwargs,

224 - )

225 - hf_processor = self.info.get_hf_processor(**mm_kwargs)

226 - image_processor = hf_processor.image_processor

227 -

228 - # HF processor pops the `num_patches` kwarg, which is needed by vLLM

229 - if (images := mm_data.get("images")) is not None:

230 - mm_items = self.info.parse_mm_data({"image": images}, validate=False)

231 - parsed_images = mm_items.get_items("image", ImageProcessorItems)

232 - image_sizes = [

233 - parsed_images.get_image_size(i) for i in range(len(parsed_images))

234 - ]

235 -

236 - num_patches = [

237 - self.info.get_num_patches(

238 - image_width=image_size.width,

239 - image_height=image_size.height,

240 - size=image_processor.size,

241 - min_patches=image_processor.min_patches,

242 - max_patches=image_processor.max_patches,

243 - )

244 - for image_size in image_sizes

245 - ]

246 - processed_outputs["num_patches"] = torch.tensor(num_patches)

247 -

248 - return processed_outputs

249 -

250 - def _get_mm_fields_config(

251 - self,

252 - hf_inputs: BatchFeature,

253 - hf_processor_mm_kwargs: Mapping[str, object],

254 - ) -> Mapping[str, MultiModalFieldConfig]:

255 - num_patches = hf_inputs.get("num_patches", torch.empty(0))

256 - return dict(

257 - pixel_values=MultiModalFieldConfig.flat_from_sizes("image", num_patches),

258 - num_patches=MultiModalFieldConfig.batched("image"),

259 - image_embeds=MultiModalFieldConfig.batched("image"),

260 - )

261 -

262 - def _get_prompt_updates(

263 - self,

264 - mm_items: MultiModalDataItems,

265 - hf_processor_mm_kwargs: Mapping[str, object],

266 - out_mm_kwargs: MultiModalKwargsItems,

267 - ) -> Sequence[PromptUpdate]:

268 - hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)

269 - image_token = hf_processor.image_token

270 - img_patch_token = hf_processor.img_patch_token

271 - image_processor = hf_processor.image_processor

272 -

273 - def get_replacement(item_idx: int):

274 - images = mm_items.get_items("image", ImageProcessorItems)

275 - image_size: ImageSize = images.get_image_size(item_idx)

276 - num_patches = self.info.get_num_patches(

277 - image_width=image_size.width,

278 - image_height=image_size.height,

279 - size=image_processor.size,

280 - min_patches=image_processor.min_patches,

281 - max_patches=image_processor.max_patches,

282 - )

283 - repl = hf_processor._prompt_split_image(num_patches=num_patches)

284 -

285 - return PromptUpdateDetails.select_text(repl, img_patch_token)

286 -

287 - return [

288 - PromptReplacement(

289 - modality="image",

290 - target=image_token,

291 - replacement=get_replacement,

292 - )

293 - ]

294 -

295 -

296 -def _get_num_hidden_layers(hf_config: AyaVisionConfig) -> int:

297 - feature_layers = hf_config.vision_feature_layer

298 - num_hidden_layers = hf_config.vision_config.num_hidden_layers

299 - # If we have one feature layer, initialize up to that layer

300 - if isinstance(feature_layers, int):

301 - return get_layer_index(feature_layers, num_hidden_layers)

302 - # If we have multiple feature layers, initialize up to the deepest m

303 - elif isinstance(feature_layers, (list, tuple)):

304 - return max(get_layer_index(idx, num_hidden_layers) for idx in feature_layers)

305 - raise TypeError(

306 - f"vision_layer_feature type: {type(feature_layers)} is not supported"

307 - )

308 -

309 -

310 -@MULTIMODAL_REGISTRY.register_processor(

311 - AyaVisionMultiModalProcessor,

312 - info=AyaVisionProcessingInfo,

313 - dummy_inputs=AyaVisionDummyInputsBuilder,

314 -)

315 -class AyaVisionForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):

316 - hf_to_vllm_mapper = WeightsMapper(

317 - orig_to_new_prefix={

318 - # mapping for new names in checkpoint saved after transformers v4.52

319 - "model.language_model.": "language_model.model.",

320 - "model.vision_tower.": "vision_tower.",

321 - "model.multi_modal_projector.": "multi_modal_projector.",

322 - "lm_head.": "language_model.lm_head.",

323 - }

324 - )

325 -

326 - @classmethod

327 - def get_placeholder_str(cls, modality: str, i: int) -> str | None:

328 - if modality.startswith("image"):

329 - return "<image>"

330 -

331 - raise ValueError("Only image modality is supported")

332 -

333 - def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):

334 - super().__init__()

335 - config: AyaVisionConfig = vllm_config.model_config.hf_config

336 - quant_config = vllm_config.quant_config

337 - multimodal_config = vllm_config.model_config.multimodal_config

338 - num_hidden_layers = _get_num_hidden_layers(config)

339 - self.config = config

340 - self.quant_config = quant_config

341 - self.multimodal_config = multimodal_config

342 -

343 - with self._mark_tower_model(vllm_config, "image"):

344 - self.vision_tower = SiglipVisionModel(

345 - config.vision_config,

346 - quant_config,

347 - num_hidden_layers_override=num_hidden_layers,

348 - prefix=maybe_prefix(prefix, "vision_model"),

349 - )

350 - self.multi_modal_projector = AyaVisionMultiModalProjector(config)

351 -

352 - with self._mark_language_model(vllm_config):

353 - self.language_model = init_vllm_registered_model(

354 - vllm_config=vllm_config,

355 - hf_config=config.text_config,

356 - prefix=maybe_prefix(prefix, "model"),

357 - # Cohere2ForCausalLM and CohereForCausalLM are the same on vllm

358 - architectures=["Cohere2ForCausalLM"],

359 - )

360 -

361 - self.make_empty_intermediate_tensors = (

362 - self.language_model.make_empty_intermediate_tensors

363 - )

364 -

365 - @property

366 - def dtype(self):

367 - return next(self.parameters()).dtype

368 -

369 - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:

370 - loader = AutoWeightsLoader(self)

371 - return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)

372 -

373 - def _image_pixels_to_features(

374 - self,

375 - vision_tower: SiglipVisionModel,

376 - pixel_values: torch.Tensor,

377 - ) -> torch.Tensor | tuple[torch.Tensor, ...]:

378 - return vision_tower(

379 - pixel_values.to(dtype=vision_tower.dtype),

380 - feature_select_strategy=self.config.vision_feature_select_strategy,

381 - )

382 -

383 - def _process_image_input(

384 - self, image_input: AyaVisionImagePixelInputs, **kwargs

385 - ) -> list[torch.Tensor]:

386 - pixel_values = image_input["pixel_values"]

387 - num_patches = image_input["num_patches"]

388 - image_features = self._image_pixels_to_features(

389 - self.vision_tower, pixel_values=pixel_values

390 - )

391 - image_embeds = self.multi_modal_projector(image_features)

392 - return [e.flatten(0, 2) for e in image_embeds.split(num_patches.tolist())]

393 -

394 - def _parse_and_validate_image_input(

395 - self, **kwargs: object

396 - ) -> AyaVisionImagePixelInputs | None:

397 - pixel_values = kwargs.pop("pixel_values", None)

398 - num_patches = kwargs.pop("num_patches", None)

399 - image_embeds = kwargs.pop("image_embeds", None)

400 - assert image_embeds is None, "Aya Vision does not support image_embeds."

401 -

402 - if pixel_values is None:

403 - return None

404 -

405 - return AyaVisionImagePixelInputs(

406 - type="pixel_values",

407 - pixel_values=pixel_values,

408 - num_patches=num_patches,

409 - resolve_bindings={

410 - "h": self.config.vision_config.image_size,

411 - "w": self.config.vision_config.image_size,

412 - },

413 - )

414 -

415 - def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:

416 - image_input = self._parse_and_validate_image_input(**kwargs)

417 - if image_input is None:

418 - return []

419 -

420 - return self._process_image_input(image_input, **kwargs)

421 -

422 - def forward(

423 - self,

424 - input_ids: torch.Tensor | None,

425 - positions: torch.Tensor,

426 - intermediate_tensors: IntermediateTensors | None = None,

427 - inputs_embeds: torch.Tensor | None = None,

428 - **kwargs: object,

429 - ) -> torch.Tensor | IntermediateTensors:

430 - if intermediate_tensors is not None:

431 - inputs_embeds = None

432 -

433 - hidden_states = self.language_model.model(

434 - input_ids=input_ids,

435 - positions=positions,

436 - intermediate_tensors=intermediate_tensors,

437 - inputs_embeds=inputs_embeds,

438 - )

439 - return hidden_states

440 -

441 - def compute_logits(

442 - self,

443 - hidden_states: torch.Tensor,

444 - ) -> torch.Tensor | None:

445 - return self.language_model.compute_logits(hidden_states)

vllm/model_executor/models/musicflamingo.py deleted

-449

@@ -1,449 +0,0 @@

1 -# SPDX-License-Identifier: Apache-2.0

2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -

6 -# reserved.

7 -#

8 -# Licensed under the Apache License, Version 2.0 (the "License");

9 -# you may not use this file except in compliance with the License.

10 -# You may obtain a copy of the License at

11 -#

12 -# http://www.apache.org/licenses/LICENSE-2.0

13 -#

14 -# Unless required by applicable law or agreed to in writing, software

15 -# distributed under the License is distributed on an "AS IS" BASIS,

16 -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

17 -# See the License for the specific language governing permissions and

18 -# limitations under the License.

19 -

20 -from collections.abc import Callable, Mapping, Sequence

21 -from math import pi

22 -from typing import Annotated, Any, Optional, TypeAlias

23 -

24 -import torch

25 -from torch import Tensor, broadcast_tensors, nn

26 -from transformers import BatchFeature

27 -from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS

28 -from transformers.models.musicflamingo import (

29 - MusicFlamingoConfig,

30 - MusicFlamingoProcessor,

31 -)

32 -

33 -from vllm.config import VllmConfig

34 -from vllm.config.multimodal import BaseDummyOptions

35 -from vllm.inputs import MultiModalDataDict

36 -from vllm.multimodal import MULTIMODAL_REGISTRY

37 -from vllm.multimodal.inputs import (

38 - MultiModalFieldConfig,

39 - MultiModalKwargsItems,

40 -)

41 -from vllm.multimodal.parse import (

42 - DictEmbeddingItems,

43 - ModalityData,

44 - ModalityDataItems,

45 - MultiModalDataItems,

46 - MultiModalDataParser,

47 -)

48 -from vllm.multimodal.processing import (

49 - PromptReplacement,

50 - PromptUpdate,

51 - PromptUpdateDetails,

52 -)

53 -from vllm.utils.tensor_schema import TensorShape

54 -

55 -from .audioflamingo3 import (

56 - AudioFlamingo3DummyInputsBuilder,

57 - AudioFlamingo3EmbeddingInputs,

58 - AudioFlamingo3Encoder,

59 - AudioFlamingo3FeatureInputs,

60 - AudioFlamingo3ForConditionalGeneration,

61 - AudioFlamingo3MultiModalDataParser,

62 - AudioFlamingo3MultiModalProcessor,

63 - AudioFlamingo3MultiModalProjector,

64 - AudioFlamingo3ProcessingInfo,

65 - _audioflamingo3_field_config,

66 - _count_audio_tokens_from_mask,

67 -)

68 -

69 -

70 -def rotate_half(x):

71 - x = x.reshape(*x.shape[:-1], -1, 2)

72 - x1, x2 = x.unbind(dim=-1)

73 - x = torch.stack((-x2, x1), dim=-1)

74 - return x.flatten(-2)

75 -

76 -

77 -def apply_rotary_time_emb(hidden_states, cos, sin):

78 - original_dtype = hidden_states.dtype

79 - hidden_states = hidden_states.to(torch.float64)

80 - cos = cos.to(hidden_states)

81 - sin = sin.to(hidden_states)

82 - rot_dim = cos.shape[-1]

83 - if rot_dim > hidden_states.shape[-1]:

84 - raise ValueError(

85 - f"feature dimension {hidden_states.shape[-1]} is not of "

86 - f"sufficient size to rotate in all the positions {rot_dim}"

87 - )

88 -

89 - rotated = hidden_states[..., :rot_dim]

90 - passthrough = hidden_states[..., rot_dim:]

91 - rotated = (rotated * cos) + (rotate_half(rotated) * sin)

92 - return torch.cat((rotated, passthrough), dim=-1).to(original_dtype)

93 -

94 -

95 -class MusicFlamingoRotaryEmbedding(nn.Module):

96 - inv_freq: torch.Tensor

97 -

98 - def __init__(self, config: MusicFlamingoConfig, device=None):

99 - super().__init__()

100 - self.max_seq_len_cached = config.max_position_embeddings

101 - self.original_max_seq_len = config.max_position_embeddings

102 -

103 - self.config = config

104 - self.rope_type = self.config.rope_parameters["rope_type"]

105 - rope_init_fn: Callable = self.compute_default_rope_parameters

106 - if self.rope_type != "default":

107 - rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]

108 - inv_freq, self.attention_scaling = rope_init_fn(self.config, device)

109 -

110 - self.register_buffer("inv_freq", inv_freq, persistent=False)

111 - self.register_buffer("original_inv_freq", inv_freq.clone(), persistent=False)

112 - position_angles = self._compute_position_angles(self.inv_freq)

113 - self.register_buffer("position_angles", position_angles, persistent=False)

114 -

115 - @staticmethod

116 - def compute_default_rope_parameters(

117 - config: MusicFlamingoConfig | None = None,

118 - device: Optional["torch.device"] = None,

119 - seq_len: int | None = None,

120 - ) -> tuple["torch.Tensor", float]:

121 - del seq_len

122 - base = config.rope_parameters["rope_theta"]

123 - partial_rotary_factor = config.rope_parameters.get("partial_rotary_factor", 1.0)

124 - head_dim = getattr(config, "head_dim", None) or (

125 - config.hidden_size // config.num_attention_heads

126 - )

127 - dim = int(head_dim * partial_rotary_factor)

128 - attention_factor = 1.0

129 -

130 - inv_freq = 1.0 / (

131 - base

132 - ** (

133 - torch.arange(0, dim, 2, dtype=torch.int64).to(

134 - device=device,

135 - dtype=torch.float,

136 - )

137 - / dim

138 - )

139 - )

140 - return inv_freq, attention_factor

141 -

142 - def _compute_position_angles(self, inv_freq):

143 - positions = torch.arange(

144 - int(self.max_seq_len_cached),

145 - device=inv_freq.device,

146 - dtype=inv_freq.dtype,

147 - )

148 - positions = positions / self.max_seq_len_cached * (2 * pi)

149 - position_angles = positions.unsqueeze(-1) * inv_freq

150 - position_angles = torch.repeat_interleave(position_angles, 2, dim=-1)

151 - return position_angles.to(dtype=inv_freq.dtype)

152 -

153 - def _restore_fp32_rope_buffers(self) -> None:

154 - rope_init_fn: Callable = self.compute_default_rope_parameters

155 - if self.rope_type != "default":

156 - rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]

157 - inv_freq, self.attention_scaling = rope_init_fn(

158 - self.config, self.inv_freq.device

159 - )

160 -

161 - self.inv_freq = inv_freq

162 - self.original_inv_freq = inv_freq.clone()

163 - self.position_angles = self._compute_position_angles(inv_freq)

164 -

165 - def _apply(self, fn):

166 - super()._apply(fn)

167 - self._restore_fp32_rope_buffers()

168 - return self

169 -

170 - @torch.no_grad()

171 - def forward(self, timestamps: Tensor, seq_len: int) -> tuple[Tensor, Tensor]:

172 - window_starts = timestamps[:, 0].to(

173 - device=self.inv_freq.device,

174 - dtype=self.inv_freq.dtype,

175 - )

176 - window_duration = self.config.audio_frame_step * 4 * seq_len

177 - window_positions = (

178 - torch.round(window_starts / window_duration) / self.max_seq_len_cached

179 - )

180 - window_freqs = window_positions.unsqueeze(-1) * self.inv_freq

181 - window_freqs = torch.repeat_interleave(window_freqs, 2, dim=-1)

182 -

183 - window_freqs = window_freqs[:, None, :]

184 - time_freqs = self.position_angles[:seq_len][None, :, :]

185 - window_freqs, time_freqs = broadcast_tensors(window_freqs, time_freqs)

186 - freqs = torch.cat((window_freqs, time_freqs), dim=-1)

187 - angle = (-timestamps * 2 * pi).to(freqs)

188 - freqs = freqs * angle.unsqueeze(-1)

189 - return freqs.cos(), freqs.sin()

190 -

191 -

192 -class MusicFlamingoFeatureInputs(AudioFlamingo3FeatureInputs):

193 - rote_timestamps: Annotated[

194 - torch.Tensor | None,

195 - TensorShape(

196 - "num_chunks",

197 - "num_audio_time_steps",

198 - dynamic_dims={"num_audio_time_steps"},

199 - ),

200 - ]

201 -

202 -

203 -MusicFlamingoEmbeddingInputs = AudioFlamingo3EmbeddingInputs

204 -

205 -MusicFlamingoInputs: TypeAlias = (

206 - MusicFlamingoFeatureInputs | MusicFlamingoEmbeddingInputs

207 -)

208 -

209 -

210 -class MusicFlamingoEncoder(AudioFlamingo3Encoder):

211 - pass

212 -

213 -

214 -class MusicFlamingoMultiModalProjector(AudioFlamingo3MultiModalProjector):

215 - pass

216 -

217 -

218 -class MusicFlamingoProcessingInfo(AudioFlamingo3ProcessingInfo):

219 - def get_hf_config(self) -> MusicFlamingoConfig:

220 - return self.ctx.get_hf_config(MusicFlamingoConfig)

221 -

222 - def get_hf_processor(self, **kwargs: object) -> MusicFlamingoProcessor:

223 - return self.ctx.get_hf_processor(MusicFlamingoProcessor, **kwargs)

224 -

225 - def get_data_parser(self) -> MultiModalDataParser:

226 - feature_extractor = self.get_feature_extractor()

227 - return MusicFlamingoMultiModalDataParser(

228 - target_sr=feature_extractor.sampling_rate,

229 - audio_resample_method="soxr",

230 - expected_hidden_size=self._get_expected_hidden_size(),

231 - )

232 -

233 - def get_supported_mm_limits(self) -> Mapping[str, int | None]:

234 - return {"audio": 1}

235 -

236 -

237 -class MusicFlamingoDummyInputsBuilder(AudioFlamingo3DummyInputsBuilder):

238 - def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:

239 - num_audios = mm_counts.get("audio", 0)

240 - hf_processor = self.info.get_hf_processor()

241 - return hf_processor.audio_token * num_audios

242 -

243 - def get_dummy_mm_data(

244 - self,

245 - seq_len: int,

246 - mm_counts: Mapping[str, int],

247 - mm_options: Mapping[str, BaseDummyOptions],

248 - ) -> MultiModalDataDict:

249 - hf_processor = self.info.get_hf_processor()

250 - feature_extractor = self.info.get_feature_extractor()

251 - sampling_rate = feature_extractor.sampling_rate

252 - audio_len = int(hf_processor.max_audio_len * sampling_rate)

253 - num_audios = mm_counts.get("audio", 0)

254 - audio_overrides = mm_options.get("audio")

255 -

256 - return {

257 - "audio": self._get_dummy_audios(

258 - length=audio_len,

259 - num_audios=num_audios,

260 - overrides=audio_overrides,

261 - )

262 - }

263 -

264 -

265 -def _musicflamingo_field_config(hf_inputs: Mapping[str, torch.Tensor]):

266 - fields = dict(_audioflamingo3_field_config(hf_inputs))

267 - chunk_counts = hf_inputs.get("chunk_counts")

268 - if chunk_counts is not None:

269 - fields["rote_timestamps"] = MultiModalFieldConfig.flat_from_sizes(

270 - "audio", chunk_counts, dim=0

271 - )

272 - else:

273 - fields["rote_timestamps"] = MultiModalFieldConfig.batched("audio")

274 - return fields

275 -

276 -

277 -class MusicFlamingoMultiModalDataParser(AudioFlamingo3MultiModalDataParser):

278 - def _parse_audio_data(

279 - self,

280 - data: dict[str, torch.Tensor] | ModalityData[Any],

281 - ) -> ModalityDataItems[Any, Any] | None:

282 - if isinstance(data, dict):

283 - return DictEmbeddingItems(

284 - data,

285 - modality="audio",

286 - required_fields={"audio_embeds"},

287 - fields_factory=_musicflamingo_field_config,

288 - )

289 - return super()._parse_audio_data(data)

290 -

291 -

292 -class MusicFlamingoMultiModalProcessor(AudioFlamingo3MultiModalProcessor):

293 - def _get_mm_fields_config(

294 - self,

295 - hf_inputs: BatchFeature,

296 - hf_processor_mm_kwargs: Mapping[str, object],

297 - ) -> Mapping[str, MultiModalFieldConfig]:

298 - return _musicflamingo_field_config(hf_inputs)

299 -

300 - def _get_prompt_updates(

301 - self,

302 - mm_items: MultiModalDataItems,

303 - hf_processor_mm_kwargs: Mapping[str, object],

304 - out_mm_kwargs: MultiModalKwargsItems,

305 - ) -> Sequence[PromptUpdate]:

306 - processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)

307 - tokenizer = self.info.get_tokenizer()

308 - vocab = tokenizer.get_vocab()

309 -

310 - audio_token = processor.audio_token

311 - audio_token_id = vocab.get(audio_token, processor.audio_token_id)

312 -

313 - audio_bos_token = processor.audio_bos_token

314 - audio_bos_token_id = vocab.get(audio_bos_token, processor.audio_bos_token_id)

315 -

316 - audio_eos_token = processor.audio_eos_token

317 - audio_eos_token_id = vocab.get(audio_eos_token, processor.audio_eos_token_id)

318 -

319 - out_mm_data = out_mm_kwargs.get_data()

320 - feature_attention_mask = out_mm_data.get("feature_attention_mask")

321 - chunk_counts = out_mm_data.get("chunk_counts")

322 -

323 - def get_replacement_musicflamingo(item_idx: int):

324 - if feature_attention_mask is not None:

325 - num_features = _count_audio_tokens_from_mask(

326 - feature_attention_mask,

327 - chunk_counts,

328 - item_idx,

329 - )

330 - else:

331 - audio_embeds = out_mm_data["audio_embeds"][item_idx]

332 - num_features = audio_embeds.shape[0]

333 -

334 - if num_features == 0:

335 - raise ValueError("Audio is too short")

336 -

337 - full_tokens = [

338 - audio_bos_token_id,

339 - *([audio_token_id] * int(num_features)),

340 - audio_eos_token_id,

341 - ]

342 -

343 - return PromptUpdateDetails.select_token_id(

344 - full_tokens,

345 - embed_token_id=audio_token_id,

346 - )

347 -

348 - return [

349 - PromptReplacement(

350 - modality="audio",

351 - target=audio_token,

352 - replacement=get_replacement_musicflamingo,

353 - )

354 - ]

355 -

356 -

357 -@MULTIMODAL_REGISTRY.register_processor(

358 - MusicFlamingoMultiModalProcessor,

359 - info=MusicFlamingoProcessingInfo,

360 - dummy_inputs=MusicFlamingoDummyInputsBuilder,

361 -)

362 -class MusicFlamingoForConditionalGeneration(AudioFlamingo3ForConditionalGeneration):

363 - """vLLM MusicFlamingo model aligned with HF modular_musicflamingo."""

364 -

365 - def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):

366 - super().__init__(vllm_config=vllm_config, prefix=prefix)

367 - self.audio_tower = MusicFlamingoEncoder(self.config.audio_config)

368 - self.multi_modal_projector = MusicFlamingoMultiModalProjector(self.config)

369 - self.pos_emb = MusicFlamingoRotaryEmbedding(self.config)

370 -

371 - def _parse_and_validate_audio_input(

372 - self, **kwargs: object

373 - ) -> MusicFlamingoInputs | None:

374 - rote_timestamps = kwargs.pop("rote_timestamps", None)

375 - audio_input = super()._parse_and_validate_audio_input(**kwargs)

376 - if audio_input is None or audio_input["type"] == "audio_embeds":

377 - return audio_input

378 -

379 - return MusicFlamingoFeatureInputs(

380 - type="audio_features",

381 - input_features=audio_input["input_features"],

382 - feature_attention_mask=audio_input["feature_attention_mask"],

383 - chunk_counts=audio_input["chunk_counts"],

384 - rote_timestamps=rote_timestamps,

385 - )

386 -

387 - def _build_audio_timestamps(

388 - self,

389 - chunk_counts: list[int],

390 - seq_len: int,

391 - device: torch.device,

392 - ) -> torch.Tensor:

393 - audio_embed_frame_step = self.config.audio_frame_step * 4

394 - frame_offsets = (

395 - torch.arange(seq_len, device=device, dtype=torch.float32)

396 - * audio_embed_frame_step

397 - )

398 -

399 - if not chunk_counts:

400 - return frame_offsets.new_empty((0, seq_len))

401 -

402 - window_indices = torch.cat(

403 - [

404 - torch.arange(count, device=device, dtype=torch.float32)

405 - for count in chunk_counts

406 - ]

407 - )

408 - return (

409 - window_indices.unsqueeze(1) * seq_len * audio_embed_frame_step

410 - + frame_offsets

411 - )

412 -

413 - def _process_audio_input(

414 - self, audio_input: MusicFlamingoInputs

415 - ) -> torch.Tensor | tuple[torch.Tensor, ...]:

416 - if audio_input["type"] == "audio_embeds":

417 - return super()._process_audio_input(audio_input)

418 -

419 - rote_timestamps = audio_input["rote_timestamps"]

420 - (

421 - input_features,

422 - feature_attention_mask,

423 - chunk_counts,

424 - ) = self._normalize_audio_feature_inputs(audio_input)

425 - hidden_states = self._encode_audio_features(

426 - input_features,

427 - feature_attention_mask,

428 - )

429 - if rote_timestamps is None:

430 - rote_timestamps = self._build_audio_timestamps(

431 - chunk_counts,

432 - seq_len=hidden_states.shape[-2],

433 - device=hidden_states.device,

434 - )

435 - elif isinstance(rote_timestamps, list):

436 - rote_timestamps = torch.cat(rote_timestamps, dim=0)

437 -

438 - cos, sin = self.pos_emb(

439 - rote_timestamps.to(hidden_states.device),

440 - seq_len=hidden_states.shape[-2],

441 - )

442 - hidden_states = apply_rotary_time_emb(hidden_states, cos, sin)

443 - audio_features = self.multi_modal_projector(hidden_states)

444 -

445 - return self._group_audio_embeddings(

446 - audio_features,

447 - feature_attention_mask,

448 - chunk_counts,

449 - )

vllm/model_executor/models/registry.py

+2 -8

@@ -337,14 +337,6 @@ _MULTIMODAL_MODELS = {

337 "audioflamingo3",

338 "AudioFlamingo3ForConditionalGeneration",

339 ),

340 - "MusicFlamingoForConditionalGeneration": (

341 - "musicflamingo",

342 - "MusicFlamingoForConditionalGeneration",

343 - ),

344 - "AyaVisionForConditionalGeneration": (

345 - "aya_vision",

346 - "AyaVisionForConditionalGeneration",

347 - ),

340 "BagelForConditionalGeneration": ("bagel", "BagelForConditionalGeneration"),

341 "BeeForConditionalGeneration": ("bee", "BeeForConditionalGeneration"),

342 "Blip2ForConditionalGeneration": ("blip2", "Blip2ForConditionalGeneration"),

@@ -733,6 +725,8 @@ _PREVIOUSLY_SUPPORTED_MODELS = {

725 "TarsierForConditionalGeneration": "0.24.0",

726 "Tarsier2ForConditionalGeneration": "0.23.0", # last version with Transformers v4

727 "MantisForConditionalGeneration": "0.24.0",

728 + "MusicFlamingoForConditionalGeneration": "0.24.0",

729 + "AyaVisionForConditionalGeneration": "0.24.0",

730 }

731

732 _OOT_SUPPORTED_MODELS = {

vllm/transformers_utils/model_arch_config_convertor.py

+1 -1

@@ -345,7 +345,7 @@ class ModelArchConfigConvertorBase:

345 max_len_key = key

346 derived_max_model_len = min(derived_max_model_len, max_len)

347

348 - # For Command-R / Cohere, Cohere2 / Aya Vision models

348 + # For Command-R / Cohere, Cohere2 models

349 if tmp_max_len := getattr(self.hf_text_config, "model_max_length", None):

350 max_len_key = "model_max_length"

351 derived_max_model_len = tmp_max_len