@tduncan / Nvidia-Optimization-Stack-VLLM / commits / fe26c705f7

[Model] Add Bailing V3 VL support (#55921)

Signed-off-by: zexplorerhj <zhjoneson@163.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

zexplorerhj committed Sep 10, 2026 at 17:46 UTC fe26c705f7099da3b155b3cb6218f013bb308e63
19 files changed +1825 -20
docs/models/supported_models.md
+1
@@ -538,6 +538,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
538 | `AriaForConditionalGeneration` | Aria | T + I<sup>+</sup> | `rhymes-ai/Aria` | | |
539 | `AudioFlamingo3ForConditionalGeneration` | AudioFlamingo3 | T + A | `nvidia/audio-flamingo-3-hf`, `nvidia/music-flamingo-hf` | ✅︎ | ✅︎ |
540 | `BagelForConditionalGeneration` | BAGEL | T + I<sup>+</sup> | `ByteDance-Seed/BAGEL-7B-MoT` | ✅︎ | ✅︎ |
541 +| `BailingMoeV3VLForConditionalGeneration` | Ling 3.0 Flash VL | T + I<sup>E+</sup> | `inclusionAI/Ling-3.0-flash-VL` | | ✅︎ |
542 | `BeeForConditionalGeneration` | Bee-8B | T + I<sup>E+</sup> | `Open-Bee/Bee-8B-RL`, `Open-Bee/Bee-8B-SFT` | | ✅︎ |
543 | `Blip2ForConditionalGeneration` | BLIP-2 | T + I<sup>E</sup> | `Salesforce/blip2-opt-2.7b`, `Salesforce/blip2-opt-6.7b`, etc. | ✅︎ | ✅︎ |
544 | `Cohere2VisionForConditionalGeneration` | Command A Vision, Command-A+ | T + I<sup>+</sup> | `CohereLabs/command-a-vision-07-2025`, `CohereLabs/command-a-plus-05-2026`, etc. | | ✅︎ |
tests/model_executor/test_bailing_mrope.py new
+176
@@ -0,0 +1,176 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import copy
5 +from types import SimpleNamespace
6 +
7 +import pytest
8 +import torch
9 +
10 +from vllm.config import DeviceConfig, VllmConfig, set_current_vllm_config
11 +from vllm.model_executor.layers.rotary_embedding import get_rope
12 +from vllm.model_executor.layers.rotary_embedding.bailing_mrope import (
13 + BailingMRotaryEmbedding,
14 +)
15 +from vllm.model_executor.models.bailing_moe_v3 import _build_mla_rotary_embedding
16 +from vllm.platforms import current_platform
17 +
18 +
19 +def test_bailing_mrope_selects_alternating_spatial_then_temporal():
20 + with set_current_vllm_config(VllmConfig(device_config=DeviceConfig("cpu"))):
21 + rotary_emb = BailingMRotaryEmbedding(
22 + head_size=12,
23 + rotary_dim=12,
24 + max_position_embeddings=8,
25 + base=10_000,
26 + is_neox_style=False,
27 + dtype=torch.float32,
28 + mrope_section=[2, 2, 2],
29 + )
30 +
31 + positions = torch.tensor([[1, 2], [3, 4], [5, 6]])
32 + frequencies = torch.arange(6).expand(8, -1)
33 + position_values = torch.arange(8).unsqueeze(1) * 10
34 + cos = position_values + frequencies
35 + sin = cos + 100
36 + cos_sin_cache = torch.cat((cos, sin), dim=-1)
37 +
38 + actual_cos, actual_sin = rotary_emb.select_cos_sin(
39 + positions,
40 + cos_sin_cache,
41 + )
42 +
43 + expected_cos = torch.tensor(
44 + [
45 + [30, 51, 32, 53, 14, 15],
46 + [40, 61, 42, 63, 24, 25],
47 + ]
48 + )
49 + torch.testing.assert_close(actual_cos, expected_cos)
50 + torch.testing.assert_close(actual_sin, expected_cos + 100)
51 +
52 +
53 +def test_bailing_factory_preserves_mla_width_and_config():
54 + config = SimpleNamespace(
55 + max_position_embeddings=128,
56 + rope_parameters={
57 + "rope_type": "default",
58 + "rope_theta": 6_000_000,
59 + "partial_rotary_factor": 0.5,
60 + "mrope_section": [8, 12, 12],
61 + },
62 + rope_scaling={"partial_rotary_factor": 0.5},
63 + )
64 + original = copy.deepcopy(vars(config))
65 + with set_current_vllm_config(VllmConfig(device_config=DeviceConfig("cpu"))):
66 + rope = _build_mla_rotary_embedding(config, head_size=64)
67 + assert rope is _build_mla_rotary_embedding(config, head_size=64)
68 + generic = get_rope(
69 + head_size=64,
70 + max_position=128,
71 + is_neox_style=False,
72 + rope_parameters={
73 + "rope_type": "default",
74 + "rope_theta": 6_000_000,
75 + "mrope_section": [8, 12, 12],
76 + },
77 + )
78 + assert isinstance(rope, BailingMRotaryEmbedding)
79 + assert rope is not generic
80 + assert rope.rotary_dim == 64
81 + assert vars(config) == original
82 +
83 +
84 +def test_bailing_factory_rejects_unsupported_scaling():
85 + config = SimpleNamespace(
86 + rope_parameters={
87 + "rope_type": "yarn",
88 + "factor": 2.0,
89 + "mrope_section": [8, 12, 12],
90 + }
91 + )
92 + with pytest.raises(ValueError, match="only supports rope_type='default'"):
93 + _build_mla_rotary_embedding(config, head_size=64)
94 +
95 +
96 +@pytest.mark.skipif(
97 + not current_platform.is_cuda_alike(),
98 + reason="Skipping CUDA/ROCm-only test.",
99 +)
100 +@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
101 +@pytest.mark.parametrize("position_kind", ["text", "multimodal"])
102 +@torch.inference_mode()
103 +def test_bailing_mrope_cuda_matches_native_for_strided_mla_views(
104 + dtype: torch.dtype,
105 + position_kind: str,
106 +):
107 + device = torch.device("cuda")
108 + num_tokens = 17
109 + num_query_heads = 8
110 + with set_current_vllm_config(VllmConfig()):
111 + rotary_emb = BailingMRotaryEmbedding(
112 + head_size=64,
113 + rotary_dim=64,
114 + max_position_embeddings=64,
115 + base=6_000_000,
116 + is_neox_style=False,
117 + dtype=dtype,
118 + mrope_section=[8, 12, 12],
119 + ).to(device)
120 +
121 + torch.manual_seed(0)
122 + multimodal_positions = torch.randint(
123 + 0,
124 + 64,
125 + (3, 2 * num_tokens),
126 + dtype=torch.int32,
127 + device=device,
128 + )[:, ::2]
129 + if position_kind == "multimodal":
130 + positions = multimodal_positions
131 + reference_positions = positions
132 + else:
133 + positions = torch.randint(
134 + 0,
135 + 64,
136 + (2 * num_tokens,),
137 + dtype=torch.int64,
138 + device=device,
139 + )[::2]
140 + reference_positions = positions.expand(3, -1)
141 + query_buffer = torch.randn(
142 + num_tokens,
143 + num_query_heads,
144 + 192,
145 + dtype=dtype,
146 + device=device,
147 + )
148 + key_buffer = torch.randn(
149 + num_tokens,
150 + 576,
151 + dtype=dtype,
152 + device=device,
153 + )
154 + query = query_buffer[..., 128:]
155 + key = key_buffer[:, 512:].unsqueeze(1)
156 + assert not query.is_contiguous()
157 + assert not key.is_contiguous()
158 +
159 + query_prefix = query_buffer[..., :128].clone()
160 + key_prefix = key_buffer[:, :512].clone()
161 + expected_query, expected_key = rotary_emb.forward_native(
162 + reference_positions,
163 + query.clone(),
164 + key.clone(),
165 + )
166 + actual_query, actual_key = rotary_emb.forward_cuda(positions, query, key)
167 + assert expected_key is not None
168 + assert actual_key is not None
169 +
170 + torch.testing.assert_close(actual_query, expected_query, atol=1e-2, rtol=1.6e-2)
171 + torch.testing.assert_close(actual_key, expected_key, atol=1e-2, rtol=1.6e-2)
172 + assert torch.equal(query_buffer[..., :128], query_prefix)
173 + assert torch.equal(key_buffer[:, :512], key_prefix)
174 + if position_kind == "multimodal":
175 + assert actual_query.data_ptr() == query.data_ptr()
176 + assert actual_key.data_ptr() == key.data_ptr()
tests/models/multimodal/processing/test_bailing_moe_v3_vl.py new
+126
@@ -0,0 +1,126 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""Bailing prompt expansion tests without downloading a checkpoint."""
4 +
5 +from types import SimpleNamespace
6 +
7 +import pytest
8 +import torch
9 +from PIL import Image
10 +from tokenizers import Tokenizer, decoders, models, pre_tokenizers
11 +from transformers import PreTrainedTokenizerFast
12 +
13 +from vllm.model_executor.models.bailing_moe_v3_vl import (
14 + IMAGE_PLACEHOLDER,
15 + IMAGE_TOKEN,
16 + BailingMoeV3VLMultiModalProcessor,
17 +)
18 +from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems
19 +from vllm.multimodal.parse import ImageProcessorItems, MultiModalDataItems
20 +
21 +pytestmark = pytest.mark.cpu_test
22 +
23 +
24 +@pytest.fixture
25 +def processor():
26 + vocab = {
27 + token: index
28 + for index, token in enumerate(sorted(pre_tokenizers.ByteLevel.alphabet()))
29 + }
30 + # A newline added by the processor must merge with an existing newline.
31 + vocab["ĊĊ"] = len(vocab)
32 + backend = Tokenizer(models.BPE(vocab=vocab, merges=[("Ċ", "Ċ")]))
33 + backend.pre_tokenizer = pre_tokenizers.ByteLevel(
34 + add_prefix_space=False, use_regex=False
35 + )
36 + backend.decoder = decoders.ByteLevel()
37 + tokenizer = PreTrainedTokenizerFast(
38 + tokenizer_object=backend,
39 + additional_special_tokens=[
40 + "<|vision_start|>",
41 + IMAGE_TOKEN,
42 + "<|vision_end|>",
43 + ],
44 + clean_up_tokenization_spaces=False,
45 + )
46 + processor = object.__new__(BailingMoeV3VLMultiModalProcessor)
47 + processor.info = SimpleNamespace(
48 + get_tokenizer=lambda: tokenizer,
49 + get_hf_config=lambda: SimpleNamespace(
50 + vision_config=SimpleNamespace(spatial_merge_size=2)
51 + ),
52 + )
53 + return processor
54 +
55 +
56 +@pytest.mark.parametrize(
57 + ("prompt", "expected_text", "token_counts"),
58 + [
59 + (
60 + f"before{IMAGE_PLACEHOLDER}after",
61 + f"before<|vision_start|>{IMAGE_TOKEN * 2}<|vision_end|>\nafter",
62 + [2],
63 + ),
64 + (
65 + f"before{IMAGE_TOKEN}after",
66 + f"before<|vision_start|>{IMAGE_TOKEN * 2}<|vision_end|>\nafter",
67 + [2],
68 + ),
69 + (
70 + f"before{IMAGE_PLACEHOLDER}\nafter",
71 + f"before<|vision_start|>{IMAGE_TOKEN * 2}<|vision_end|>\n\nafter",
72 + [2],
73 + ),
74 + (
75 + f"before{IMAGE_PLACEHOLDER}{IMAGE_PLACEHOLDER}after",
76 + f"before<|vision_start|>{IMAGE_TOKEN * 2}<|vision_end|>\n"
77 + f"<|vision_start|>{IMAGE_TOKEN * 3}<|vision_end|>\nafter",
78 + [2, 3],
79 + ),
80 + ],
81 + ids=["wrapped", "bare", "existing_newline", "adjacent_images"],
82 +)
83 +def test_image_expansion_matches_wrappers_and_newline_tokenization(
84 + processor, prompt, expected_text, token_counts
85 +):
86 + tokenizer = processor.info.get_tokenizer()
87 + image_id = tokenizer.get_vocab()[IMAGE_TOKEN]
88 + grids = torch.tensor([[1, 2, 2 * count] for count in token_counts])
89 + mm_kwargs = MultiModalKwargsItems.from_hf_inputs(
90 + {"image_grid_thw": grids},
91 + {"image_grid_thw": MultiModalFieldConfig.batched("image")},
92 + )
93 + mm_items = MultiModalDataItems(
94 + {"image": ImageProcessorItems([Image.new("RGB", (1, 1))] * len(token_counts))}
95 + )
96 + updates = processor._get_mm_prompt_updates(mm_items, {}, mm_kwargs)
97 + prompt_ids = tokenizer.encode(prompt, add_special_tokens=False)
98 + expected_ids = tokenizer.encode(expected_text, add_special_tokens=False)
99 +
100 + actual_ids, placeholders = processor._apply_prompt_updates(prompt_ids, updates)
101 +
102 + assert actual_ids == expected_ids
103 + assert len(placeholders["image"]) == len(token_counts)
104 + found = processor._find_mm_placeholders(actual_ids, updates)
105 + search_offset = 0
106 + for index, count in enumerate(token_counts):
107 + placeholder = placeholders["image"][index]
108 + expected_offset = expected_ids.index(image_id, search_offset)
109 + assert placeholder.start_idx == expected_offset
110 + assert placeholder.tokens == [image_id] * count
111 + assert placeholder.to_range().get_num_embeds() == count
112 + assert placeholder.to_range() == found["image"][index].to_range()
113 + search_offset = expected_offset + count
114 +
115 +
116 +def test_text_only_prompt_preserves_original_token_ids(processor):
117 + tokenizer = processor.info.get_tokenizer()
118 + prompt_ids = tokenizer.encode("before\n", add_special_tokens=False)
119 + prompt_ids += tokenizer.encode("\nafter", add_special_tokens=False)
120 + # Deliberately use two newline tokens instead of their merged BPE token.
121 + assert prompt_ids != tokenizer.encode("before\n\nafter", add_special_tokens=False)
122 +
123 + actual_ids, placeholders = processor._apply_prompt_updates(prompt_ids, {})
124 +
125 + assert actual_ids == prompt_ids
126 + assert placeholders == {}
tests/models/multimodal/test_mapping.py
+30
@@ -184,6 +184,36 @@ def test_cosmos3_edge_checkpoint_weights_mapper():
184 )
185
186
187 +@pytest.mark.cpu_test
188 +def test_bailing_vl_mapper_handles_module_and_parameter_names():
189 + from vllm.model_executor.models.bailing_moe_v3_vl import (
190 + BailingMoeV3VLForConditionalGeneration,
191 + )
192 +
193 + mapper = BailingMoeV3VLForConditionalGeneration.hf_to_vllm_mapper
194 + names = {
195 + "model.visual": "visual",
196 + "model.visual.blocks.0.attn.qkv.weight": "visual.blocks.0.attn.qkv.weight",
197 + "model.linear_proj": "linear_proj",
198 + "linear_proj.0": "linear_proj.linear_fc1",
199 + "model.linear_proj.0.weight": "linear_proj.linear_fc1.weight",
200 + "linear_proj.2": "linear_proj.linear_fc2",
201 + "linear_proj.2.weight_scale": "linear_proj.linear_fc2.weight_scale",
202 + "model.linear_proj.2.bias": "linear_proj.linear_fc2.bias",
203 + "linear_proj.linear_fc1.weight": "linear_proj.linear_fc1.weight",
204 + "model.linear_proj.linear_fc2.weight": "linear_proj.linear_fc2.weight",
205 + "model.layers.0.attention.q_proj.weight": (
206 + "language_model.model.layers.0.self_attn.q_proj.weight"
207 + ),
208 + "model.layers.0.mlp.experts.3.up_proj.weight": (
209 + "language_model.model.layers.0.mlp.experts.3.up_proj.weight"
210 + ),
211 + "lm_head": "language_model.lm_head",
212 + "lm_head.weight": "language_model.lm_head.weight",
213 + }
214 + assert mapper.apply_list(list(names)) == list(names.values())
215 +
216 +
217 def create_repo_dummy_weights(repo: str) -> Iterable[tuple[str, torch.Tensor]]:
218 """Create weights from safetensors checkpoint metadata"""
219 metadata = try_get_safetensors_metadata(repo)
tests/models/registry.py
+12
@@ -796,6 +796,18 @@ _MULTIMODAL_EXAMPLE_MODELS = {
796 },
797 ),
798 "BagelForConditionalGeneration": _HfExamplesInfo("ByteDance-Seed/BAGEL-7B-MoT"),
799 + "BailingMoeV3VLForConditionalGeneration": _HfExamplesInfo(
800 + "inclusionAI/Ling-3.0-flash-VL",
801 + trust_remote_code=True,
802 + is_available_online=True,
803 + use_original_num_layers=True,
804 + hf_overrides={
805 + "text_config": {
806 + "num_hidden_layers": 6,
807 + "layer_types": ["linear_attention"] * 5 + ["full_attention"],
808 + }
809 + },
810 + ),
811 "BeeForConditionalGeneration": _HfExamplesInfo(
812 "Open-Bee/Bee-8B-RL",
813 trust_remote_code=True,
tests/parser/engine/test_ling3.py
+7 -6
@@ -3,6 +3,8 @@
3
4 import json
5
6 +import pytest
7 +
8 from tests.parser.engine.replay_harness import MockTokenizer, _test_request
9 from vllm.reasoning import ReasoningParserManager
10 from vllm.tool_parsers import ToolParserManager
@@ -57,19 +59,18 @@ def test_ling3_disable_thinking_keeps_reasoning_as_content():
59 assert content == "<think>reason</think>answer"
60
61
60 -def test_ling3_enable_thinking_keeps_open_reasoning_as_content():
62 +@pytest.mark.parametrize("model_output", ["only reasoning", "<think>only reasoning"])
63 +def test_ling3_truncated_reasoning_stays_reasoning(model_output):
64 parser_cls = ReasoningParserManager.get_reasoning_parser("ling3")
65 parser = parser_cls(
66 _tokenizer(),
67 chat_template_kwargs={"enable_thinking": True},
68 )
69
67 - reasoning, content = parser.extract_reasoning(
68 - "<think>only reasoning", _test_request()
69 - )
70 + reasoning, content = parser.extract_reasoning(model_output, _test_request())
71
71 - assert reasoning is None
72 - assert content == "only reasoning"
72 + assert reasoning == "only reasoning"
73 + assert content is None
74
75
76 def test_ling3_tool_call_without_newline():
tests/transformers_utils/test_bailing_moe_v3_vl_config.py new
+159
@@ -0,0 +1,159 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import copy
5 +import json
6 +from pathlib import Path
7 +from typing import Any
8 +
9 +import pytest
10 +from transformers import AutoConfig
11 +
12 +from vllm.transformers_utils.config import get_config, uses_mrope
13 +from vllm.transformers_utils.configs.bailing_moe_v3_vl import (
14 + BailingMoeV3TextConfig,
15 + BailingMoeV3VisionConfig,
16 + BailingMoeV3VLConfig,
17 +)
18 +
19 +
20 +def _bailing_v3_vl_config() -> dict[str, Any]:
21 + return {
22 + "architectures": ["BailingMoeV3VLForConditionalGeneration"],
23 + "auto_map": {
24 + "AutoConfig": ("configuration_bailing_moe_v3_vl.BailingMoeV3VLConfig")
25 + },
26 + "model_type": "bailing_moe_v3_vl",
27 + "mrope_section": [8, 12, 12],
28 + "text_config": {
29 + "num_hidden_layers": 42,
30 + "layer_group_size": 6,
31 + "rope_theta": 6_000_000,
32 + },
33 + "vision_config": {"disable_merger_proj": True},
34 + }
35 +
36 +
37 +def _write_config(path: Path, config: dict[str, Any]) -> None:
38 + path.mkdir()
39 + (path / "config.json").write_text(json.dumps(config), encoding="utf-8")
40 +
41 +
42 +def test_bailing_v3_vl_config_loads_without_remote_code(tmp_path: Path):
43 + model_path = tmp_path / "model"
44 + _write_config(model_path, _bailing_v3_vl_config())
45 +
46 + config = get_config(model_path, trust_remote_code=False)
47 +
48 + assert isinstance(config, BailingMoeV3VLConfig)
49 + assert isinstance(config.text_config, BailingMoeV3TextConfig)
50 + assert isinstance(config.vision_config, BailingMoeV3VisionConfig)
51 + assert isinstance(
52 + AutoConfig.from_pretrained(model_path, trust_remote_code=False),
53 + BailingMoeV3VLConfig,
54 + )
55 + assert config.mrope_section == [8, 12, 12]
56 + assert uses_mrope(config)
57 +
58 + text_config = config.text_config
59 + assert text_config.model_type == "bailing_hybrid"
60 + assert text_config.architectures == ["BailingMoeV3ForCausalLM"]
61 + assert text_config.rope_parameters["mrope_section"] == [8, 12, 12]
62 + assert text_config.rope_parameters["rope_theta"] == 6_000_000
63 + assert text_config.layer_types.count("linear_attention") == 35
64 + assert [
65 + layer_idx
66 + for layer_idx, layer_type in enumerate(text_config.layer_types)
67 + if layer_type == "full_attention"
68 + ] == [5, 11, 17, 23, 29, 35, 41]
69 + assert config.vision_config.disable_merger_proj is True
70 +
71 + normalized = config.to_dict()
72 + restored = BailingMoeV3VLConfig.from_dict(normalized)
73 + assert isinstance(restored.text_config, BailingMoeV3TextConfig)
74 + assert isinstance(restored.vision_config, BailingMoeV3VisionConfig)
75 + assert restored.to_dict() == normalized
76 +
77 +
78 +def test_bailing_v3_vl_config_preserves_explicit_nested_values():
79 + config_dict = _bailing_v3_vl_config()
80 + config_dict["architectures"] = ["CustomBailingVLForConditionalGeneration"]
81 + text_config = config_dict["text_config"]
82 + text_config["architectures"] = ["CustomBailingForCausalLM"]
83 + text_config["layer_types"] = (["linear_attention"] * 5 + ["full_attention"]) * 7
84 + text_config["rope_theta"] = 1_000_000
85 + text_config["rope_parameters"] = {
86 + "rope_type": "default",
87 + "rope_theta": 1_000_000,
88 + "mrope_section": [8, 12, 12],
89 + }
90 + original = copy.deepcopy(config_dict)
91 +
92 + config = BailingMoeV3VLConfig(**config_dict)
93 +
94 + assert config_dict == original
95 + assert config.architectures == ["CustomBailingVLForConditionalGeneration"]
96 + assert config.text_config.architectures == ["CustomBailingForCausalLM"]
97 + assert config.text_config.layer_types == text_config["layer_types"]
98 + assert config.text_config.rope_parameters["rope_theta"] == 1_000_000
99 + assert config.text_config.rope_parameters["mrope_section"] == [8, 12, 12]
100 +
101 +
102 +def test_bailing_v3_vl_config_normalizes_config_instance():
103 + text_config = BailingMoeV3TextConfig()
104 +
105 + config = BailingMoeV3VLConfig(
106 + text_config=text_config,
107 + mrope_section=[8, 12, 12],
108 + )
109 +
110 + assert config.text_config is text_config
111 + assert text_config.rope_parameters["mrope_section"] == [8, 12, 12]
112 +
113 +
114 +def test_bailing_v3_text_config_trailing_and_legacy_layers():
115 + expected = ["linear_attention", "linear_attention", "full_attention"] * 2 + [
116 + "full_attention"
117 + ] * 2
118 + text_config = BailingMoeV3TextConfig(
119 + num_hidden_layers=8,
120 + layer_group_size=3,
121 + )
122 + assert text_config.layer_types == expected
123 +
124 + text_config = BailingMoeV3TextConfig(
125 + num_hidden_layers=8,
126 + layer_group_size=3,
127 + layers_block_type=["mamba", "mamba", "attention"] * 2 + ["attention"] * 2,
128 + )
129 + assert text_config.layer_types == expected
130 +
131 +
132 +def test_bailing_v3_text_config_accepts_legacy_rope_type():
133 + text_config = BailingMoeV3TextConfig(rope_scaling={"type": "linear", "factor": 2.0})
134 +
135 + assert text_config.rope_parameters["type"] == "linear"
136 + assert text_config.rope_parameters["rope_type"] == "linear"
137 +
138 +
139 +def test_bailing_v3_vl_config_rejects_invalid_values():
140 + with pytest.raises(ValueError, match="layer_group_size must be positive"):
141 + BailingMoeV3TextConfig(layer_group_size=0)
142 +
143 + with pytest.raises(ValueError, match="must match the layer_group_size schedule"):
144 + BailingMoeV3TextConfig(
145 + num_hidden_layers=8,
146 + layer_group_size=3,
147 + layer_types=["full_attention"] * 8,
148 + )
149 +
150 + with pytest.raises(ValueError, match="conflicting M-RoPE sections"):
151 + BailingMoeV3VLConfig(
152 + text_config={
153 + "rope_parameters": {
154 + "rope_type": "default",
155 + "mrope_section": [4, 14, 14],
156 + }
157 + },
158 + mrope_section=[8, 12, 12],
159 + )
vllm/config/speculative.py
+12 -1
@@ -847,7 +847,18 @@ class SpeculativeConfig:
847 )
848
849 architectures = getattr(hf_config, "architectures", []) or []
850 - if initial_architecture == "BailingMoeV3ForCausalLM":
850 + if initial_architecture == "BailingMoeV3VLForConditionalGeneration":
851 + quantization_config = getattr(hf_config, "quantization_config", None)
852 + hf_config = copy.deepcopy(hf_config.text_config)
853 + if (
854 + quantization_config is not None
855 + and getattr(hf_config, "quantization_config", None) is None
856 + ):
857 + hf_config.quantization_config = copy.deepcopy(quantization_config)
858 + if initial_architecture in (
859 + "BailingMoeV3ForCausalLM",
860 + "BailingMoeV3VLForConditionalGeneration",
861 + ):
862 hf_config.model_type = "bailing_hybrid_v3_mtp"
863 elif (
864 hf_config.model_type == "bailing_hybrid"
vllm/model_executor/layers/rotary_embedding/__init__.py
+11
@@ -6,6 +6,7 @@ from typing import Any
6
7 import torch
8
9 +from .bailing_mrope import BailingMRotaryEmbedding
10 from .base import RotaryEmbedding
11 from .deepseek_scaling_rope import (
12 DeepseekScalingRotaryEmbedding,
@@ -97,6 +98,16 @@ def get_rope(
98 dtype,
99 **extra_kwargs,
100 )
101 + elif scaling_type == "bailing_mrope":
102 + rotary_emb = BailingMRotaryEmbedding(
103 + head_size,
104 + rotary_dim,
105 + max_position,
106 + base,
107 + is_neox_style,
108 + dtype,
109 + mrope_section=rope_parameters["mrope_section"],
110 + )
111 elif scaling_type == "default":
112 if "mrope_section" in rope_parameters:
113 rotary_emb = MRotaryEmbedding(
vllm/model_executor/layers/rotary_embedding/bailing_mrope.py new
+351
@@ -0,0 +1,351 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import torch
5 +
6 +from vllm.triton_utils import tl, triton
7 +
8 +from .mrope import MRotaryEmbedding
9 +
10 +
11 +def _validate_bailing_mrope_config(
12 + head_size: int,
13 + rotary_dim: int,
14 + mrope_section: list[int],
15 +) -> int:
16 + if rotary_dim <= 0 or rotary_dim % 2 != 0 or rotary_dim > head_size:
17 + raise ValueError(
18 + "Bailing M-RoPE requires an even rotary_dim in (0, head_size], "
19 + f"got rotary_dim={rotary_dim} and head_size={head_size}"
20 + )
21 + if len(mrope_section) != 3:
22 + raise ValueError(
23 + "Bailing M-RoPE expects [T, H, W] sections, "
24 + f"got mrope_section={mrope_section}"
25 + )
26 +
27 + temporal_size, height_size, width_size = mrope_section
28 + if any(section_size < 0 for section_size in mrope_section):
29 + raise ValueError(
30 + "Bailing M-RoPE sections must be non-negative, "
31 + f"got mrope_section={mrope_section}"
32 + )
33 + if height_size != width_size:
34 + raise ValueError(
35 + "Bailing M-RoPE requires equal H and W sections, "
36 + f"got height_size={height_size} and width_size={width_size}"
37 + )
38 + spatial_size = height_size + width_size
39 + if spatial_size + temporal_size != rotary_dim // 2:
40 + raise ValueError(
41 + "Bailing M-RoPE sections must sum to rotary_dim // 2, got "
42 + f"mrope_section={mrope_section} and rotary_dim={rotary_dim}"
43 + )
44 + return spatial_size
45 +
46 +
47 +def _validate_bailing_positions(
48 + positions: torch.Tensor,
49 + *,
50 + require_2d: bool = False,
51 +) -> None:
52 + if positions.dtype not in (torch.int32, torch.int64):
53 + raise ValueError(
54 + "Bailing M-RoPE positions must have int32 or int64 dtype, got "
55 + f"dtype={positions.dtype}"
56 + )
57 + if positions.ndim == 1 and not require_2d:
58 + return
59 + if positions.ndim == 2:
60 + if positions.shape[0] == 3:
61 + return
62 + raise ValueError(
63 + "Bailing M-RoPE expects T/H/W positions with shape [3, N], "
64 + f"got shape={tuple(positions.shape)}"
65 + )
66 + if require_2d:
67 + raise ValueError(
68 + "Bailing M-RoPE expects T/H/W positions with shape [3, N], "
69 + f"got shape={tuple(positions.shape)}"
70 + )
71 + raise ValueError(
72 + f"Bailing M-RoPE positions must be 1D or 2D, got shape={tuple(positions.shape)}"
73 + )
74 +
75 +
76 +@triton.jit
77 +def _triton_bailing_mrope_forward(
78 + q_ptr,
79 + k_ptr,
80 + positions_ptr,
81 + cos_sin_cache_ptr,
82 + q_stride_token,
83 + q_stride_head,
84 + q_stride_dim,
85 + k_stride_token,
86 + k_stride_head,
87 + k_stride_dim,
88 + positions_stride_axis,
89 + positions_stride_token,
90 + cache_stride_position,
91 + cache_stride_dim,
92 + n_qh: tl.constexpr,
93 + n_kh: tl.constexpr,
94 + rotary_dim: tl.constexpr,
95 + pad_n_qh: tl.constexpr,
96 + pad_n_kh: tl.constexpr,
97 + pad_half_rotary_dim: tl.constexpr,
98 + spatial_size: tl.constexpr,
99 +):
100 + """Apply GPT-J-style H/W-interleaved, T-tail M-RoPE in place."""
101 + token_idx = tl.program_id(0)
102 + position_offset = token_idx * positions_stride_token
103 + temporal_position = tl.load(positions_ptr + position_offset).to(tl.int64)
104 + height_position = tl.load(
105 + positions_ptr + positions_stride_axis + position_offset
106 + ).to(tl.int64)
107 + width_position = tl.load(
108 + positions_ptr + 2 * positions_stride_axis + position_offset
109 + ).to(tl.int64)
110 +
111 + frequency_offsets = tl.arange(0, pad_half_rotary_dim)
112 + frequency_mask = frequency_offsets < rotary_dim // 2
113 + spatial_position = tl.where(
114 + frequency_offsets % 2 == 0,
115 + height_position,
116 + width_position,
117 + )
118 + selected_position = tl.where(
119 + frequency_offsets < spatial_size,
120 + spatial_position,
121 + temporal_position,
122 + )
123 + cache_offsets = (
124 + selected_position * cache_stride_position + frequency_offsets * cache_stride_dim
125 + )
126 + cos_row = tl.load(
127 + cos_sin_cache_ptr + cache_offsets,
128 + mask=frequency_mask,
129 + other=0.0,
130 + ).to(tl.float32)
131 + sin_row = tl.load(
132 + cos_sin_cache_ptr + cache_offsets + (rotary_dim // 2) * cache_stride_dim,
133 + mask=frequency_mask,
134 + other=0.0,
135 + ).to(tl.float32)
136 +
137 + q_head_offsets = tl.arange(0, pad_n_qh)[:, None]
138 + k_head_offsets = tl.arange(0, pad_n_kh)[:, None]
139 + pair_offsets = frequency_offsets[None, :]
140 + q_mask = (q_head_offsets < n_qh) & frequency_mask[None, :]
141 + k_mask = (k_head_offsets < n_kh) & frequency_mask[None, :]
142 +
143 + q_base = q_ptr + token_idx * q_stride_token
144 + q_even_offsets = q_head_offsets * q_stride_head + 2 * pair_offsets * q_stride_dim
145 + q_odd_offsets = q_even_offsets + q_stride_dim
146 + q_even = tl.load(q_base + q_even_offsets, mask=q_mask, other=0.0).to(tl.float32)
147 + q_odd = tl.load(q_base + q_odd_offsets, mask=q_mask, other=0.0).to(tl.float32)
148 + q_even_cos = (q_even * cos_row[None, :]).to(q_ptr.type.element_ty).to(tl.float32)
149 + q_odd_sin = (q_odd * sin_row[None, :]).to(q_ptr.type.element_ty).to(tl.float32)
150 + q_odd_cos = (q_odd * cos_row[None, :]).to(q_ptr.type.element_ty).to(tl.float32)
151 + q_even_sin = (q_even * sin_row[None, :]).to(q_ptr.type.element_ty).to(tl.float32)
152 + tl.store(
153 + q_base + q_even_offsets,
154 + q_even_cos - q_odd_sin,
155 + mask=q_mask,
156 + )
157 + tl.store(
158 + q_base + q_odd_offsets,
159 + q_odd_cos + q_even_sin,
160 + mask=q_mask,
161 + )
162 +
163 + k_base = k_ptr + token_idx * k_stride_token
164 + k_even_offsets = k_head_offsets * k_stride_head + 2 * pair_offsets * k_stride_dim
165 + k_odd_offsets = k_even_offsets + k_stride_dim
166 + k_even = tl.load(k_base + k_even_offsets, mask=k_mask, other=0.0).to(tl.float32)
167 + k_odd = tl.load(k_base + k_odd_offsets, mask=k_mask, other=0.0).to(tl.float32)
168 + k_even_cos = (k_even * cos_row[None, :]).to(k_ptr.type.element_ty).to(tl.float32)
169 + k_odd_sin = (k_odd * sin_row[None, :]).to(k_ptr.type.element_ty).to(tl.float32)
170 + k_odd_cos = (k_odd * cos_row[None, :]).to(k_ptr.type.element_ty).to(tl.float32)
171 + k_even_sin = (k_even * sin_row[None, :]).to(k_ptr.type.element_ty).to(tl.float32)
172 + tl.store(
173 + k_base + k_even_offsets,
174 + k_even_cos - k_odd_sin,
175 + mask=k_mask,
176 + )
177 + tl.store(
178 + k_base + k_odd_offsets,
179 + k_odd_cos + k_even_sin,
180 + mask=k_mask,
181 + )
182 +
183 +
184 +def triton_bailing_mrope(
185 + query: torch.Tensor,
186 + key: torch.Tensor,
187 + positions: torch.Tensor,
188 + cos_sin_cache: torch.Tensor,
189 + spatial_size: int,
190 + head_size: int,
191 + rotary_dim: int,
192 +) -> tuple[torch.Tensor, torch.Tensor]:
193 + """Fused Bailing M-RoPE for packed or strided Q/K."""
194 + _validate_bailing_positions(positions, require_2d=True)
195 + num_tokens = positions.shape[-1]
196 + if num_tokens == 0:
197 + return query, key
198 +
199 + query_3d = query.view(num_tokens, -1, head_size)
200 + key_3d = key.view(num_tokens, -1, head_size)
201 + n_q_heads = query_3d.shape[1]
202 + n_k_heads = key_3d.shape[1]
203 + pad_n_q_heads = triton.next_power_of_2(n_q_heads)
204 + pad_n_k_heads = triton.next_power_of_2(n_k_heads)
205 + pad_half_rotary_dim = triton.next_power_of_2(rotary_dim // 2)
206 + _triton_bailing_mrope_forward[(num_tokens,)](
207 + query_3d,
208 + key_3d,
209 + positions,
210 + cos_sin_cache,
211 + query_3d.stride(0),
212 + query_3d.stride(1),
213 + query_3d.stride(2),
214 + key_3d.stride(0),
215 + key_3d.stride(1),
216 + key_3d.stride(2),
217 + positions.stride(0),
218 + positions.stride(1),
219 + cos_sin_cache.stride(0),
220 + cos_sin_cache.stride(1),
221 + n_q_heads,
222 + n_k_heads,
223 + rotary_dim,
224 + pad_n_q_heads,
225 + pad_n_k_heads,
226 + pad_half_rotary_dim,
227 + spatial_size,
228 + num_warps=4,
229 + )
230 + return query, key
231 +
232 +
233 +class BailingMRotaryEmbedding(MRotaryEmbedding):
234 + """Bailing M-RoPE with alternating H/W frequencies and a temporal tail."""
235 +
236 + def __init__(
237 + self,
238 + head_size: int,
239 + rotary_dim: int,
240 + max_position_embeddings: int,
241 + base: float,
242 + is_neox_style: bool,
243 + dtype: torch.dtype,
244 + mrope_section: list[int],
245 + ) -> None:
246 + self.spatial_size = _validate_bailing_mrope_config(
247 + head_size, rotary_dim, mrope_section
248 + )
249 + super().__init__(
250 + head_size,
251 + rotary_dim,
252 + max_position_embeddings,
253 + base,
254 + is_neox_style,
255 + dtype,
256 + mrope_section=mrope_section,
257 + )
258 +
259 + def select_cos_sin(
260 + self,
261 + positions: torch.Tensor,
262 + cos_sin_cache: torch.Tensor | None = None,
263 + ) -> tuple[torch.Tensor, torch.Tensor]:
264 + _validate_bailing_positions(positions)
265 + if cos_sin_cache is None:
266 + cos_sin_cache = self.cos_sin_cache
267 + cos_sin = cos_sin_cache[positions.to(torch.long)]
268 + cos, sin = cos_sin.chunk(2, dim=-1)
269 + if positions.ndim == 1:
270 + return cos, sin
271 +
272 + spatial_mask = torch.arange(self.spatial_size, device=cos.device) % 2 == 0
273 + spatial_mask = spatial_mask.unsqueeze(0)
274 + spatial_cos = torch.where(
275 + spatial_mask,
276 + cos[1, :, : self.spatial_size],
277 + cos[2, :, : self.spatial_size],
278 + )
279 + spatial_sin = torch.where(
280 + spatial_mask,
281 + sin[1, :, : self.spatial_size],
282 + sin[2, :, : self.spatial_size],
283 + )
284 + temporal_slice = slice(self.spatial_size, self.rotary_dim // 2)
285 + return (
286 + torch.cat((spatial_cos, cos[0, :, temporal_slice]), dim=-1),
287 + torch.cat((spatial_sin, sin[0, :, temporal_slice]), dim=-1),
288 + )
289 +
290 + def forward_native(
291 + self,
292 + positions: torch.Tensor,
293 + query: torch.Tensor,
294 + key: torch.Tensor | None = None,
295 + offsets: torch.Tensor | None = None,
296 + ) -> tuple[torch.Tensor, torch.Tensor | None]:
297 + if key is None:
298 + raise ValueError("Bailing M-RoPE requires key states")
299 +
300 + cos_sin_cache = self._match_cos_sin_cache_dtype(query)
301 + cos, sin = self.select_cos_sin(positions, cos_sin_cache)
302 + num_tokens = positions.shape[-1]
303 + query_shape = query.shape
304 + query = query.view(num_tokens, -1, self.head_size)
305 + query_rot = self.apply_rotary_emb.forward_native(
306 + query[..., : self.rotary_dim],
307 + cos,
308 + sin,
309 + )
310 + query = torch.cat((query_rot, query[..., self.rotary_dim :]), dim=-1)
311 +
312 + key_shape = key.shape
313 + key = key.view(num_tokens, -1, self.head_size)
314 + key_rot = self.apply_rotary_emb.forward_native(
315 + key[..., : self.rotary_dim],
316 + cos,
317 + sin,
318 + )
319 + key = torch.cat((key_rot, key[..., self.rotary_dim :]), dim=-1)
320 + return query.reshape(query_shape), key.reshape(key_shape)
321 +
322 + def forward_cuda(
323 + self,
324 + positions: torch.Tensor,
325 + query: torch.Tensor,
326 + key: torch.Tensor | None = None,
327 + offsets: torch.Tensor | None = None,
328 + ) -> tuple[torch.Tensor, torch.Tensor | None]:
329 + if positions.ndim == 2 and not self.is_neox_style:
330 + if key is None:
331 + raise ValueError("Bailing M-RoPE requires key states")
332 + cos_sin_cache = self._match_cos_sin_cache_dtype(query)
333 + return triton_bailing_mrope(
334 + query,
335 + key,
336 + positions,
337 + cos_sin_cache,
338 + self.spatial_size,
339 + self.head_size,
340 + self.rotary_dim,
341 + )
342 + return self.forward_native(positions, query, key, offsets)
343 +
344 + def forward_cpu(
345 + self,
346 + positions: torch.Tensor,
347 + query: torch.Tensor,
348 + key: torch.Tensor | None = None,
349 + offsets: torch.Tensor | None = None,
350 + ) -> tuple[torch.Tensor, torch.Tensor | None]:
351 + return self.forward_native(positions, query, key, offsets)
vllm/model_executor/models/bailing_moe_v3.py
+23 -4
@@ -171,6 +171,27 @@ def _build_rope_parameters(config: PretrainedConfig) -> dict | None:
171 return rope_parameters or None
172
173
174 +def _build_mla_rotary_embedding(
175 + config: PretrainedConfig,
176 + head_size: int,
177 +) -> nn.Module:
178 + rope_parameters = _build_rope_parameters(config)
179 + if rope_parameters is not None and "mrope_section" in rope_parameters:
180 + rope_type = rope_parameters.get("rope_type", "default")
181 + if rope_type != "default":
182 + raise ValueError(
183 + f"Bailing M-RoPE only supports rope_type='default', got {rope_type!r}"
184 + )
185 + rope_parameters["rope_type"] = "bailing_mrope"
186 +
187 + return get_rope(
188 + head_size=head_size,
189 + max_position=getattr(config, "max_position_embeddings", 8192),
190 + is_neox_style=False,
191 + rope_parameters=rope_parameters,
192 + )
193 +
194 +
195 def _get_layer_swiglu_limit(limit_list: list | None, layer_idx: int) -> float | None:
196 if limit_list is None or layer_idx >= len(limit_list):
197 return None
@@ -524,11 +545,9 @@ class BailingMoeV3MLAAttention(nn.Module):
545 prefix=f"{prefix}.dense",
546 )
547
527 - self.rotary_emb = get_rope(
548 + self.rotary_emb = _build_mla_rotary_embedding(
549 + config,
550 head_size=self.qk_rope_head_dim,
529 - max_position=getattr(config, "max_position_embeddings", 8192),
530 - is_neox_style=False,
531 - rope_parameters=_build_rope_parameters(config),
551 )
552 mla_modules = MLAModules(
553 kv_a_layernorm=self.kv_a_layernorm,
vllm/model_executor/models/bailing_moe_v3_mtp.py
+32 -5
@@ -33,9 +33,14 @@ from vllm.model_executor.models.bailing_moe_v3 import (
33 _maybe_pad_block_fp8_shared_expert_checkpoint_tensor,
34 _maybe_remap_ling_mxfp4_weight_names,
35 )
36 -from vllm.model_executor.models.interfaces import SupportsPP
36 +from vllm.model_executor.models.interfaces import (
37 + MultiModalEmbeddings,
38 + SupportsMultiModalEmbeddings,
39 + SupportsPP,
40 +)
41 from vllm.model_executor.models.utils import (
42 PPMissingLayer,
43 + _merge_multimodal_embeddings,
44 is_pp_missing_parameter,
45 maybe_prefix,
46 )
@@ -214,8 +219,16 @@ class BailingMoeV3MultiTokenPredictor(nn.Module):
219 return self.logits_processor(head, mtp_layer.shared_head(hidden_states))
220
221
217 -@support_torch_compile
218 -class BailingMoeV3MTPModel(nn.Module, SupportsPP):
222 +@support_torch_compile(
223 + dynamic_arg_dims={
224 + "input_ids": 0,
225 + "positions": -1,
226 + "hidden_states": 0,
227 + "intermediate_tensors": 0,
228 + "inputs_embeds": 0,
229 + }
230 +)
231 +class BailingMoeV3MTPModel(nn.Module, SupportsPP, SupportsMultiModalEmbeddings):
232 hf_to_vllm_mapper = BailingMoeV3ForCausalLM.hf_to_vllm_mapper
233
234 packed_modules_mapping = {
@@ -242,8 +255,22 @@ class BailingMoeV3MTPModel(nn.Module, SupportsPP):
255 def share_lm_head(self, lm_head: nn.Module) -> None:
256 self.lm_head = lm_head
257
245 - def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
246 - return self.model.embed_input_ids(input_ids)
258 + def embed_input_ids(
259 + self,
260 + input_ids: torch.Tensor,
261 + multimodal_embeddings: MultiModalEmbeddings | None = None,
262 + *,
263 + is_multimodal: torch.Tensor | None = None,
264 + ) -> torch.Tensor:
265 + inputs_embeds = self.model.embed_input_ids(input_ids)
266 + if multimodal_embeddings is None or len(multimodal_embeddings) == 0:
267 + return inputs_embeds
268 + assert is_multimodal is not None
269 + return _merge_multimodal_embeddings(
270 + inputs_embeds=inputs_embeds,
271 + multimodal_embeddings=multimodal_embeddings,
272 + is_multimodal=is_multimodal,
273 + )
274
275 def forward(
276 self,
vllm/model_executor/models/bailing_moe_v3_vl.py new
+527
@@ -0,0 +1,527 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""Inference-only Bailing MoE V3 vision-language model."""
4 +
5 +import copy
6 +from collections.abc import Iterable, Mapping, Sequence
7 +from typing import Any
8 +
9 +import numpy as np
10 +import regex as re
11 +import torch
12 +import torch.nn as nn
13 +from transformers import AutoProcessor, BatchFeature
14 +
15 +from vllm.config import VllmConfig
16 +from vllm.config.multimodal import BaseDummyOptions
17 +from vllm.inputs import MultiModalDataDict
18 +from vllm.model_executor.layers.linear import (
19 + ColumnParallelLinear,
20 + RowParallelLinear,
21 +)
22 +from vllm.model_executor.layers.quantization import QuantizationConfig
23 +from vllm.model_executor.layers.quantization.fp8 import Fp8Config
24 +from vllm.model_executor.models.module_mapping import MultiModelKeys
25 +from vllm.multimodal import MULTIMODAL_REGISTRY
26 +from vllm.multimodal.inputs import (
27 + MultiModalFeatureSpec,
28 + MultiModalFieldConfig,
29 + MultiModalKwargsItems,
30 +)
31 +from vllm.multimodal.parse import MultiModalDataItems
32 +from vllm.multimodal.processing import (
33 + BaseDummyInputsBuilder,
34 + BaseMultiModalProcessor,
35 + PromptReplacement,
36 + PromptUpdate,
37 +)
38 +from vllm.multimodal.processing.processor import (
39 + MultiModalPromptUpdates,
40 + PlaceholderFeaturesInfo,
41 +)
42 +from vllm.sequence import IntermediateTensors
43 +
44 +from .bailing_moe_v3 import BailingMoeV3ForCausalLM
45 +from .interfaces import (
46 + HasInnerState,
47 + IsHybrid,
48 + MultiModalEmbeddings,
49 + SupportsMRoPE,
50 + SupportsMultiModal,
51 + SupportsPP,
52 + _require_is_multimodal,
53 +)
54 +from .qwen2_5_vl import (
55 + Qwen2_5_VLImageEmbeddingInputs,
56 + Qwen2_5_VLImageInputs,
57 + Qwen2_5_VLImagePixelInputs,
58 +)
59 +from .qwen2_vl import (
60 + Qwen2VLProcessingInfo,
61 + _create_qwen2vl_field_factory,
62 +)
63 +from .qwen3_vl import Qwen3_VisionTransformer
64 +from .utils import (
65 + AutoWeightsLoader,
66 + WeightsMapper,
67 + _merge_multimodal_embeddings,
68 + maybe_prefix,
69 +)
70 +
71 +IMAGE_TOKEN = "<|image_pad|>"
72 +IMAGE_PLACEHOLDER = f"<|vision_start|>{IMAGE_TOKEN}<|vision_end|>"
73 +
74 +
75 +class BailingMoeV3VLProcessingInfo(Qwen2VLProcessingInfo):
76 + """Processing metadata for the remote Bailing multimodal processor."""
77 +
78 + def get_hf_config(self):
79 + return self.ctx.get_hf_config()
80 +
81 + def get_hf_processor(self, **kwargs: object) -> AutoProcessor:
82 + # The checkpoint supplies its processor through trust_remote_code.
83 + return self.ctx.get_hf_processor(**kwargs)
84 +
85 + def get_supported_mm_limits(self) -> Mapping[str, int | None]:
86 + return {"image": None}
87 +
88 + def get_mm_max_tokens_per_item(
89 + self,
90 + seq_len: int,
91 + mm_counts: Mapping[str, int],
92 + ) -> Mapping[str, int]:
93 + return {"image": self.get_max_image_tokens()}
94 +
95 +
96 +class BailingMoeV3VLDummyInputsBuilder(
97 + BaseDummyInputsBuilder[BailingMoeV3VLProcessingInfo]
98 +):
99 + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
100 + return IMAGE_PLACEHOLDER * mm_counts.get("image", 0)
101 +
102 + def get_dummy_mm_data(
103 + self,
104 + seq_len: int,
105 + mm_counts: Mapping[str, int],
106 + mm_options: Mapping[str, BaseDummyOptions],
107 + ) -> MultiModalDataDict:
108 + target_width, target_height = self.info.get_image_size_with_most_features()
109 + return {
110 + "image": self._get_dummy_images(
111 + width=target_width,
112 + height=target_height,
113 + num_images=mm_counts.get("image", 0),
114 + overrides=mm_options.get("image"),
115 + )
116 + }
117 +
118 +
119 +class BailingMoeV3VLMultiModalProcessor(
120 + BaseMultiModalProcessor[BailingMoeV3VLProcessingInfo]
121 +):
122 + def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
123 + return self.dummy_inputs.get_dummy_text(mm_counts)
124 +
125 + def _apply_prompt_updates(
126 + self,
127 + token_ids: list[int],
128 + mm_prompt_updates: MultiModalPromptUpdates,
129 + ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
130 + if image_updates := mm_prompt_updates.get("image"):
131 + tokenizer = self.info.get_tokenizer()
132 + # HF adds a newline after each image before tokenization. Re-encode
133 + # the text so it can merge with existing whitespace at the boundary.
134 + text = re.sub(
135 + f"{re.escape(IMAGE_PLACEHOLDER)}|{re.escape(IMAGE_TOKEN)}",
136 + lambda _: IMAGE_PLACEHOLDER + "\n",
137 + tokenizer.decode(token_ids),
138 + count=len(image_updates),
139 + )
140 + token_ids = tokenizer.encode(text, add_special_tokens=False)
141 +
142 + return super()._apply_prompt_updates(token_ids, mm_prompt_updates)
143 +
144 + def _get_mm_fields_config(
145 + self,
146 + hf_inputs: BatchFeature,
147 + hf_processor_mm_kwargs: Mapping[str, object],
148 + ) -> Mapping[str, MultiModalFieldConfig]:
149 + merge_size = self.info.get_hf_config().vision_config.spatial_merge_size
150 + fields = _create_qwen2vl_field_factory(merge_size)(hf_inputs)
151 + return {
152 + key: fields[key]
153 + for key in ("pixel_values", "image_embeds", "image_grid_thw")
154 + }
155 +
156 + def _get_prompt_updates(
157 + self,
158 + mm_items: MultiModalDataItems,
159 + hf_processor_mm_kwargs: Mapping[str, Any],
160 + out_mm_kwargs: MultiModalKwargsItems,
161 + ) -> Sequence[PromptUpdate]:
162 + tokenizer = self.info.get_tokenizer()
163 + image_token_id = tokenizer.get_vocab()[IMAGE_TOKEN]
164 + merge_size = self.info.get_hf_config().vision_config.spatial_merge_size
165 + merge_unit = merge_size**2
166 +
167 + def get_image_replacement(item_idx: int) -> list[int]:
168 + out_item = out_mm_kwargs["image"][item_idx]
169 + image_grid_thw = out_item["image_grid_thw"].data
170 + assert isinstance(image_grid_thw, torch.Tensor)
171 + num_image_tokens = int(image_grid_thw.prod()) // merge_unit
172 + return [image_token_id] * num_image_tokens
173 +
174 + return [
175 + PromptReplacement(
176 + modality="image",
177 + target=[image_token_id],
178 + replacement=get_image_replacement,
179 + )
180 + ]
181 +
182 +
183 +class BailingMoeV3VisionPatchMerger(nn.Module):
184 + """Normalize patches and merge each 2x2 group without an MLP."""
185 +
186 + def __init__(
187 + self,
188 + context_dim: int,
189 + spatial_merge_size: int,
190 + norm_eps: float = 1e-6,
191 + ) -> None:
192 + super().__init__()
193 + self.hidden_size = context_dim * spatial_merge_size**2
194 + self.norm = nn.LayerNorm(context_dim, eps=norm_eps)
195 +
196 + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
197 + hidden_states = self.norm(hidden_states)
198 + return hidden_states.reshape(-1, self.hidden_size)
199 +
200 +
201 +class BailingMoeV3VisionTransformer(Qwen3_VisionTransformer):
202 + """Qwen3 ViT with Bailing's parameter-free patch merger."""
203 +
204 + def __init__(
205 + self,
206 + vision_config: Any,
207 + norm_eps: float = 1e-6,
208 + quant_config: QuantizationConfig | None = None,
209 + prefix: str = "",
210 + ) -> None:
211 + # Honor whole-tower exclusions before the parent creates linear layers.
212 + if (
213 + isinstance(quant_config, Fp8Config)
214 + and prefix in quant_config.ignored_layers
215 + ):
216 + quant_config = None
217 +
218 + vision_config = copy.deepcopy(vision_config)
219 + vision_config.deepstack_visual_indexes = []
220 + merged_hidden_size = (
221 + vision_config.hidden_size * vision_config.spatial_merge_size**2
222 + )
223 + vision_config.out_hidden_size = merged_hidden_size
224 +
225 + super().__init__(
226 + vision_config,
227 + norm_eps=norm_eps,
228 + quant_config=quant_config,
229 + prefix=prefix,
230 + )
231 + self.merger = BailingMoeV3VisionPatchMerger(
232 + context_dim=vision_config.hidden_size,
233 + spatial_merge_size=vision_config.spatial_merge_size,
234 + norm_eps=norm_eps,
235 + )
236 + self.deepstack_visual_indexes = []
237 + self.deepstack_merger_list = nn.ModuleList()
238 + self.out_hidden_size = merged_hidden_size
239 +
240 +
241 +class BailingMoeV3VLProjector(nn.Module):
242 + """Tensor-parallel MLP mapping merged vision features to text width."""
243 +
244 + def __init__(
245 + self,
246 + vision_hidden_size: int,
247 + text_hidden_size: int,
248 + quant_config: QuantizationConfig | None = None,
249 + prefix: str = "",
250 + ) -> None:
251 + super().__init__()
252 + if (
253 + isinstance(quant_config, Fp8Config)
254 + and prefix in quant_config.ignored_layers
255 + ):
256 + quant_config = None
257 +
258 + self.linear_fc1 = ColumnParallelLinear(
259 + vision_hidden_size,
260 + text_hidden_size,
261 + bias=True,
262 + quant_config=quant_config,
263 + prefix=maybe_prefix(prefix, "linear_fc1"),
264 + )
265 + self.act_fn = nn.GELU()
266 + self.linear_fc2 = RowParallelLinear(
267 + text_hidden_size,
268 + text_hidden_size,
269 + bias=True,
270 + quant_config=quant_config,
271 + prefix=maybe_prefix(prefix, "linear_fc2"),
272 + )
273 +
274 + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
275 + hidden_states, _ = self.linear_fc1(hidden_states)
276 + hidden_states = self.act_fn(hidden_states)
277 + hidden_states, _ = self.linear_fc2(hidden_states)
278 + return hidden_states
279 +
280 +
281 +@MULTIMODAL_REGISTRY.register_processor(
282 + BailingMoeV3VLMultiModalProcessor,
283 + info=BailingMoeV3VLProcessingInfo,
284 + dummy_inputs=BailingMoeV3VLDummyInputsBuilder,
285 +)
286 +class BailingMoeV3VLForConditionalGeneration(
287 + nn.Module,
288 + HasInnerState,
289 + IsHybrid,
290 + SupportsMultiModal,
291 + SupportsPP,
292 + SupportsMRoPE,
293 +):
294 + """Native vLLM wrapper for ``BailingMoeV3VLConfig`` checkpoints."""
295 +
296 + packed_modules_mapping = {
297 + **BailingMoeV3ForCausalLM.packed_modules_mapping,
298 + # The vision checkpoint already stores Q/K/V in one tensor.
299 + "qkv": ["qkv"],
300 + }
301 +
302 + hf_to_vllm_mapper = BailingMoeV3ForCausalLM.hf_to_vllm_mapper | WeightsMapper(
303 + orig_to_new_prefix={
304 + # Match module names as well as parameter names for quantization.
305 + "model.visual": "visual",
306 + "model.linear_proj": "linear_proj",
307 + "linear_proj.0": "linear_proj.linear_fc1",
308 + "linear_proj.2": "linear_proj.linear_fc2",
309 + "lm_head": "language_model.lm_head",
310 + "model.": "language_model.model.",
311 + }
312 + )
313 +
314 + @classmethod
315 + def get_placeholder_str(cls, modality: str, i: int) -> str | None:
316 + if modality.startswith("image"):
317 + return IMAGE_PLACEHOLDER
318 + raise ValueError("Only image modality is supported")
319 +
320 + @classmethod
321 + def get_mamba_state_shape_from_config(
322 + cls, vllm_config: VllmConfig
323 + ) -> tuple[tuple[int, int], tuple[int, int, int]]:
324 + text_config = vllm_config.model_config.hf_config.text_config
325 + return BailingMoeV3ForCausalLM.get_mamba_state_shape_from_config(
326 + vllm_config.with_hf_config(text_config)
327 + )
328 +
329 + @classmethod
330 + def get_mamba_state_dtype_from_config(
331 + cls, vllm_config: VllmConfig
332 + ) -> tuple[torch.dtype, torch.dtype]:
333 + text_config = vllm_config.model_config.hf_config.text_config
334 + return BailingMoeV3ForCausalLM.get_mamba_state_dtype_from_config(
335 + vllm_config.with_hf_config(text_config)
336 + )
337 +
338 + @classmethod
339 + def get_mamba_state_copy_func(cls) -> tuple:
340 + return BailingMoeV3ForCausalLM.get_mamba_state_copy_func()
341 +
342 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = "model") -> None:
343 + super().__init__()
344 + config = vllm_config.model_config.hf_config
345 + vision_config = config.vision_config
346 + text_config = config.text_config
347 + quant_config = vllm_config.quant_config
348 +
349 + self.config = config
350 + self.model_config = vllm_config.model_config
351 +
352 + merged_vision_size = vision_config.hidden_size * (
353 + vision_config.spatial_merge_size**2
354 + )
355 + with self._mark_tower_model(vllm_config, "image"):
356 + self.visual = BailingMoeV3VisionTransformer(
357 + vision_config,
358 + norm_eps=getattr(config, "rms_norm_eps", 1e-6),
359 + quant_config=quant_config,
360 + prefix=maybe_prefix(prefix, "visual"),
361 + )
362 + self.linear_proj = BailingMoeV3VLProjector(
363 + vision_hidden_size=merged_vision_size,
364 + text_hidden_size=text_config.hidden_size,
365 + quant_config=quant_config,
366 + prefix=maybe_prefix(prefix, "linear_proj"),
367 + )
368 +
369 + with self._mark_language_model(vllm_config):
370 + self.language_model = BailingMoeV3ForCausalLM(
371 + vllm_config=vllm_config.with_hf_config(text_config),
372 + prefix=maybe_prefix(prefix, "language_model"),
373 + )
374 +
375 + self.make_empty_intermediate_tensors = (
376 + self.language_model.make_empty_intermediate_tensors
377 + )
378 +
379 + def _parse_and_validate_image_input(
380 + self, **kwargs: object
381 + ) -> Qwen2_5_VLImageInputs | None:
382 + pixel_values = kwargs.pop("pixel_values", None)
383 + image_embeds = kwargs.pop("image_embeds", None)
384 + image_grid_thw = kwargs.pop("image_grid_thw", None)
385 +
386 + if pixel_values is None and image_embeds is None:
387 + return None
388 + if pixel_values is not None:
389 + return Qwen2_5_VLImagePixelInputs(
390 + type="pixel_values",
391 + pixel_values=pixel_values,
392 + image_grid_thw=image_grid_thw,
393 + )
394 + return Qwen2_5_VLImageEmbeddingInputs(
395 + type="image_embeds",
396 + image_embeds=image_embeds,
397 + image_grid_thw=image_grid_thw,
398 + )
399 +
400 + def _process_image_input(
401 + self, image_input: Qwen2_5_VLImageInputs
402 + ) -> tuple[torch.Tensor, ...]:
403 + image_grid_thw = image_input["image_grid_thw"]
404 + assert image_grid_thw.ndim == 2
405 +
406 + if image_input["type"] == "image_embeds":
407 + image_embeds = image_input["image_embeds"]
408 + else:
409 + pixel_values = image_input["pixel_values"].type(self.visual.dtype)
410 + vision_features = self.visual(
411 + pixel_values,
412 + grid_thw=image_grid_thw,
413 + )
414 + image_embeds = self.linear_proj(vision_features)
415 +
416 + image_embeds = image_embeds.to(dtype=self.visual.dtype)
417 + merge_unit = self.visual.spatial_merge_size**2
418 + sizes = (image_grid_thw.prod(-1) // merge_unit).tolist()
419 + return image_embeds.split(sizes)
420 +
421 + def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
422 + image_input = self._parse_and_validate_image_input(**kwargs)
423 + if image_input is None:
424 + return []
425 + return self._process_image_input(image_input)
426 +
427 + def embed_input_ids(
428 + self,
429 + input_ids: torch.Tensor,
430 + multimodal_embeddings: MultiModalEmbeddings | None = None,
431 + *,
432 + is_multimodal: torch.Tensor | None = None,
433 + ) -> torch.Tensor:
434 + inputs_embeds = self._embed_text_input_ids(
435 + input_ids,
436 + self.language_model.embed_input_ids,
437 + is_multimodal=is_multimodal,
438 + )
439 + if multimodal_embeddings is None or len(multimodal_embeddings) == 0:
440 + return inputs_embeds
441 +
442 + return _merge_multimodal_embeddings(
443 + inputs_embeds=inputs_embeds,
444 + multimodal_embeddings=multimodal_embeddings,
445 + is_multimodal=_require_is_multimodal(is_multimodal),
446 + )
447 +
448 + def get_mrope_input_positions(
449 + self,
450 + input_tokens: list[int],
451 + mm_features: list[MultiModalFeatureSpec],
452 + ) -> tuple[torch.Tensor, int]:
453 + position_blocks: list[np.ndarray] = []
454 + consumed = 0
455 + merge_size = self.config.vision_config.spatial_merge_size
456 +
457 + for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset):
458 + if mm_feature.modality != "image":
459 + raise ValueError(f"Unsupported modality: {mm_feature.modality}")
460 +
461 + offset = mm_feature.mm_position.offset
462 + text_len = offset - consumed
463 + start = position_blocks[-1].max() + 1 if position_blocks else 0
464 + position_blocks.append(
465 + np.broadcast_to(np.arange(text_len), (3, text_len)) + start
466 + )
467 +
468 + assert mm_feature.data is not None
469 + image_grid_thw = mm_feature.data["image_grid_thw"].data
470 + assert isinstance(image_grid_thw, torch.Tensor)
471 + t, h, w = image_grid_thw.tolist()
472 + assert t == 1, f"Image must have one temporal grid, got {t}"
473 + grid = np.indices((t, h // merge_size, w // merge_size))
474 + position_blocks.append(grid.reshape(3, -1) + text_len + start)
475 + consumed = offset + t * (h // merge_size) * (w // merge_size)
476 +
477 + if consumed < len(input_tokens):
478 + start = position_blocks[-1].max() + 1 if position_blocks else 0
479 + text_len = len(input_tokens) - consumed
480 + position_blocks.append(
481 + np.broadcast_to(np.arange(text_len), (3, text_len)) + start
482 + )
483 +
484 + if not position_blocks:
485 + positions = np.empty((3, 0), dtype=np.int64)
486 + return torch.from_numpy(positions), 0
487 +
488 + positions = np.concatenate(position_blocks, axis=1).reshape(3, -1)
489 + position_delta = int(positions.max() + 1 - len(input_tokens))
490 + return torch.from_numpy(positions), position_delta
491 +
492 + def forward(
493 + self,
494 + input_ids: torch.Tensor | None,
495 + positions: torch.Tensor,
496 + intermediate_tensors: IntermediateTensors | None = None,
497 + inputs_embeds: torch.Tensor | None = None,
498 + **kwargs: object,
499 + ) -> torch.Tensor | IntermediateTensors:
500 + if intermediate_tensors is not None:
501 + inputs_embeds = None
502 + return self.language_model(
503 + input_ids=input_ids,
504 + positions=positions,
505 + intermediate_tensors=intermediate_tensors,
506 + inputs_embeds=inputs_embeds,
507 + )
508 +
509 + def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor:
510 + return self.language_model.compute_logits(hidden_states)
511 +
512 + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
513 + loader = AutoWeightsLoader(self)
514 + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
515 +
516 + def get_mm_mapping(self) -> MultiModelKeys:
517 + return MultiModelKeys.from_string_field(
518 + language_model="language_model",
519 + connector="linear_proj",
520 + tower_model="visual.",
521 + )
522 +
523 + def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int:
524 + return num_image_tokens * self.visual.spatial_merge_size**2
525 +
526 + def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int:
527 + return num_vision_tokens // self.visual.spatial_merge_size**2
vllm/model_executor/models/registry.py
+4
@@ -350,6 +350,10 @@ _MULTIMODAL_MODELS = {
350 "AudioFlamingo3ForConditionalGeneration",
351 ),
352 "BagelForConditionalGeneration": ("bagel", "BagelForConditionalGeneration"),
353 + "BailingMoeV3VLForConditionalGeneration": (
354 + "bailing_moe_v3_vl",
355 + "BailingMoeV3VLForConditionalGeneration",
356 + ),
357 "BeeForConditionalGeneration": ("bee", "BeeForConditionalGeneration"),
358 "Blip2ForConditionalGeneration": ("blip2", "Blip2ForConditionalGeneration"),
359 "Cohere2VisionForConditionalGeneration": (
vllm/parser/ling3.py
+1 -4
@@ -71,10 +71,7 @@ class Ling3Parser(Glm47MoeParser):
71 if not self.thinking_enabled:
72 return None, model_output
73
74 - reasoning, content = super().extract_reasoning(model_output, request)
75 - if reasoning and not content and "<tool_call>" not in model_output:
76 - return None, reasoning
77 - return reasoning, content
74 + return super().extract_reasoning(model_output, request)
75
76
77 (
vllm/transformers_utils/config.py
+1
@@ -88,6 +88,7 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(
88 afmoe="AfmoeConfig",
89 axk1="AXK1Config",
90 bagel="BagelConfig",
91 + bailing_moe_v3_vl="BailingMoeV3VLConfig",
92 chatglm="ChatGLMConfig",
93 modernvbert="ColModernVBertConfig",
94 colpali="ColPaliConfig",
vllm/transformers_utils/configs/__init__.py
+6
@@ -18,6 +18,9 @@ _CLASS_TO_MODULE: dict[str, str] = {
18 "AfmoeConfig": "vllm.transformers_utils.configs.afmoe",
19 "AXK1Config": "vllm.transformers_utils.configs.AXK1",
20 "BagelConfig": "vllm.transformers_utils.configs.bagel",
21 + "BailingMoeV3TextConfig": "vllm.transformers_utils.configs.bailing_moe_v3_vl",
22 + "BailingMoeV3VisionConfig": "vllm.transformers_utils.configs.bailing_moe_v3_vl",
23 + "BailingMoeV3VLConfig": "vllm.transformers_utils.configs.bailing_moe_v3_vl",
24 "ChatGLMConfig": "vllm.transformers_utils.configs.chatglm",
25 "ColModernVBertConfig": "vllm.transformers_utils.configs.colmodernvbert",
26 "ColPaliConfig": "vllm.transformers_utils.configs.colpali",
@@ -114,6 +117,9 @@ __all__ = [
117 "AfmoeConfig",
118 "AXK1Config",
119 "BagelConfig",
120 + "BailingMoeV3TextConfig",
121 + "BailingMoeV3VisionConfig",
122 + "BailingMoeV3VLConfig",
123 "ChatGLMConfig",
124 "ColModernVBertConfig",
125 "ColPaliConfig",
vllm/transformers_utils/configs/bailing_moe_v3_vl.py new
+345
@@ -0,0 +1,345 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +# Copyright 2025 Antgroup and The HuggingFace Inc. team. All rights reserved.
4 +# Adapted from
5 +# https://huggingface.co/inclusionAI/Ling-3.0-flash/blob/51fd444268df5267074ef0b289de0e1c4cb9b381/configuration_bailing_moe_v3.py
6 +"""Bailing V3 vision-language model configuration."""
7 +
8 +from __future__ import annotations
9 +
10 +from typing import Any
11 +
12 +from transformers.configuration_utils import PretrainedConfig
13 +
14 +
15 +def _normalize_mrope_parameters(
16 + config: PretrainedConfig,
17 + mrope_section: list[int],
18 +) -> None:
19 + """Add the top-level Bailing M-RoPE split to a text config."""
20 + rope_parameters = dict(getattr(config, "rope_parameters", None) or {})
21 + text_mrope_section = rope_parameters.get("mrope_section")
22 + if text_mrope_section is not None and list(text_mrope_section) != mrope_section:
23 + raise ValueError(
24 + "Bailing V3 VL has conflicting M-RoPE sections: "
25 + f"text={text_mrope_section}, top-level={mrope_section}"
26 + )
27 +
28 + if "rope_type" not in rope_parameters and "type" not in rope_parameters:
29 + rope_parameters["rope_type"] = "default"
30 + rope_parameters.setdefault("rope_theta", getattr(config, "rope_theta", 10_000))
31 + rope_parameters["mrope_section"] = list(mrope_section)
32 + config.rope_parameters = rope_parameters
33 +
34 +
35 +def _build_layer_types(
36 + num_hidden_layers: int,
37 + layer_group_size: int,
38 +) -> list[str]:
39 + if layer_group_size <= 0:
40 + raise ValueError(
41 + f"Bailing V3 VL layer_group_size must be positive, got {layer_group_size}"
42 + )
43 +
44 + grouped_layers = num_hidden_layers // layer_group_size * layer_group_size
45 + return [
46 + "linear_attention"
47 + if ((layer_idx + 1) % layer_group_size != 0 and layer_idx < grouped_layers)
48 + else "full_attention"
49 + for layer_idx in range(num_hidden_layers)
50 + ]
51 +
52 +
53 +class BailingMoeV3TextConfig(PretrainedConfig):
54 + model_type = "bailing_hybrid"
55 + base_config_key = "text_config"
56 + keys_to_ignore_at_inference = ["past_key_values"]
57 + ignore_keys_at_rope_validation = {"mrope_section"}
58 +
59 + def __init__(
60 + self,
61 + *,
62 + vocab_size: int = 157184,
63 + hidden_size: int = 2048,
64 + intermediate_size: int = 5120,
65 + num_hidden_layers: int = 20,
66 + num_attention_heads: int = 16,
67 + num_key_value_heads: int = 4,
68 + hidden_act: str = "silu",
69 + use_qkv_bias: bool = False,
70 + use_bias: bool = False,
71 + rms_norm_eps: float = 1e-6,
72 + tie_word_embeddings: bool = False,
73 + embedding_dropout: float = 0.0,
74 + attention_dropout: float = 0.0,
75 + output_dropout: float = 0.0,
76 + initializer_range: float = 0.02,
77 + max_position_embeddings: int = 32768,
78 + rope_theta: float = 600000.0,
79 + rope_parameters: dict[str, Any] | None = None,
80 + rope_scaling: dict[str, Any] | None = None,
81 + partial_rotary_factor: float | None = None,
82 + mrope_section: list[int] | tuple[int, ...] | None = None,
83 + use_cache: bool = True,
84 + max_window_layers: int = 20,
85 + pad_token_id: int = 156892,
86 + eos_token_id: int = 156892,
87 + num_experts: int = 256,
88 + num_shared_experts: int = 1,
89 + num_experts_per_tok: int = 8,
90 + n_group: int = 8,
91 + topk_group: int = 4,
92 + moe_intermediate_size: int = 512,
93 + moe_shared_expert_intermediate_size: int = 512,
94 + first_k_dense_replace: int = 1,
95 + head_dim: int | None = 128,
96 + output_router_logits: bool = False,
97 + use_qk_norm: bool = True,
98 + num_nextn_predict_layers: int = 0,
99 + mtp_loss_scaling_factor: float = 0,
100 + moe_router_enable_expert_bias: bool = True,
101 + routed_scaling_factor: float = 1.0,
102 + layer_group_size: int = 5,
103 + layer_types: list[str] | None = None,
104 + layers_block_type: list[str] | None = None,
105 + kv_lora_rank: int = 512,
106 + q_lora_rank: int | None = None,
107 + qk_rope_head_dim: int = 64,
108 + v_head_dim: int = 128,
109 + qk_nope_head_dim: int = 128,
110 + rope_interleave: bool = True,
111 + score_function: str = "sigmoid",
112 + scoring_func: str = "sigmoid",
113 + seq_aux: bool = True,
114 + topk_method: str = "noaux_tc",
115 + router_dtype: str = "fp32",
116 + gated_attention_proj_granularity_type: str | None = None,
117 + no_kda_lora: bool = False,
118 + kda_safe_gate: bool = False,
119 + kda_lower_bound: float | None = None,
120 + short_conv_kernel_size: int = 4,
121 + architectures: list[str] | None = None,
122 + **kwargs: Any,
123 + ) -> None:
124 + kwargs.pop("model_type", None)
125 + if architectures is None:
126 + architectures = ["BailingMoeV3ForCausalLM"]
127 +
128 + if rope_parameters is not None:
129 + self.rope_parameters = dict(rope_parameters)
130 + elif rope_scaling is not None:
131 + self.rope_parameters = dict(rope_scaling)
132 + else:
133 + self.rope_parameters = {}
134 + if (
135 + "rope_type" not in self.rope_parameters
136 + and "type" not in self.rope_parameters
137 + ):
138 + self.rope_parameters["rope_type"] = "default"
139 + self.rope_parameters.setdefault("rope_theta", rope_theta)
140 + if mrope_section is not None:
141 + _normalize_mrope_parameters(self, list(mrope_section))
142 +
143 + self.vocab_size = vocab_size
144 + self.hidden_size = hidden_size
145 + self.intermediate_size = intermediate_size
146 + self.num_hidden_layers = num_hidden_layers
147 + self.num_attention_heads = num_attention_heads
148 + self.num_key_value_heads = num_key_value_heads
149 + self.hidden_act = hidden_act
150 + self.use_qkv_bias = use_qkv_bias
151 + self.use_bias = use_bias
152 + self.rms_norm_eps = rms_norm_eps
153 + self.embedding_dropout = embedding_dropout
154 + self.attention_dropout = attention_dropout
155 + self.output_dropout = output_dropout
156 + self.initializer_range = initializer_range
157 + self.max_position_embeddings = max_position_embeddings
158 + self.rope_theta = rope_theta
159 + self.partial_rotary_factor = partial_rotary_factor
160 + self.use_cache = use_cache
161 + self.max_window_layers = max_window_layers
162 + self.num_experts = num_experts
163 + self.num_shared_experts = num_shared_experts
164 + self.num_experts_per_tok = num_experts_per_tok
165 + self.n_group = n_group
166 + self.topk_group = topk_group
167 + self.moe_intermediate_size = moe_intermediate_size
168 + self.moe_shared_expert_intermediate_size = moe_shared_expert_intermediate_size
169 + self.first_k_dense_replace = first_k_dense_replace
170 + self.head_dim = head_dim or hidden_size // num_attention_heads
171 + self.output_router_logits = output_router_logits
172 + self.use_qk_norm = use_qk_norm
173 + self.num_nextn_predict_layers = num_nextn_predict_layers
174 + self.mtp_loss_scaling_factor = mtp_loss_scaling_factor
175 + self.moe_router_enable_expert_bias = moe_router_enable_expert_bias
176 + self.routed_scaling_factor = routed_scaling_factor
177 + self.layer_group_size = layer_group_size
178 + self.kv_lora_rank = kv_lora_rank
179 + self.q_lora_rank = q_lora_rank
180 + self.qk_rope_head_dim = qk_rope_head_dim
181 + self.v_head_dim = v_head_dim
182 + self.qk_nope_head_dim = qk_nope_head_dim
183 + self.qk_head_dim = qk_nope_head_dim + qk_rope_head_dim
184 + self.rope_interleave = rope_interleave
185 + self.score_function = score_function
186 + self.scoring_func = scoring_func
187 + self.seq_aux = seq_aux
188 + self.topk_method = topk_method
189 + self.router_dtype = router_dtype
190 + self.gated_attention_proj_granularity_type = (
191 + gated_attention_proj_granularity_type
192 + )
193 + self.no_kda_lora = no_kda_lora
194 + self.kda_safe_gate = kda_safe_gate
195 + self.kda_lower_bound = kda_lower_bound
196 + self.short_conv_kernel_size = short_conv_kernel_size
197 +
198 + expected_layer_types = _build_layer_types(
199 + num_hidden_layers,
200 + layer_group_size,
201 + )
202 + if layers_block_type is not None:
203 + legacy_type_mapping = {
204 + "attention": "full_attention",
205 + "mamba": "linear_attention",
206 + "conv": "linear_attention",
207 + }
208 + normalized_block_types = [
209 + legacy_type_mapping.get(layer_type, layer_type)
210 + for layer_type in layers_block_type
211 + ]
212 + if layer_types is not None and layer_types != normalized_block_types:
213 + raise ValueError(
214 + "Bailing V3 VL has conflicting layer_types and layers_block_type"
215 + )
216 + layer_types = normalized_block_types
217 + if layer_types is not None and layer_types != expected_layer_types:
218 + raise ValueError(
219 + "Bailing V3 VL layer_types must match the layer_group_size schedule"
220 + )
221 + self.layer_types = expected_layer_types
222 +
223 + super().__init__(
224 + architectures=architectures,
225 + pad_token_id=pad_token_id,
226 + eos_token_id=eos_token_id,
227 + tie_word_embeddings=tie_word_embeddings,
228 + **kwargs,
229 + )
230 + self.validate_layer_type()
231 +
232 +
233 +class BailingMoeV3VisionConfig(PretrainedConfig):
234 + model_type = "qwen3_moe_vit"
235 + base_config_key = "vision_config"
236 +
237 + def __init__(
238 + self,
239 + *,
240 + depth: int = 27,
241 + hidden_size: int = 1152,
242 + hidden_act: str = "gelu_pytorch_tanh",
243 + intermediate_size: int = 4304,
244 + num_heads: int = 16,
245 + in_channels: int = 3,
246 + patch_size: int = 16,
247 + spatial_merge_size: int = 2,
248 + temporal_patch_size: int = 2,
249 + out_hidden_size: int = 3584,
250 + num_position_embeddings: int = 2304,
251 + deepstack_visual_indexes: list[int] | None = None,
252 + initializer_range: float = 0.02,
253 + **kwargs: Any,
254 + ) -> None:
255 + kwargs.pop("model_type", None)
256 + super().__init__(**kwargs)
257 + self.depth = depth
258 + self.hidden_size = hidden_size
259 + self.hidden_act = hidden_act
260 + self.intermediate_size = intermediate_size
261 + self.num_heads = num_heads
262 + self.in_channels = in_channels
263 + self.patch_size = patch_size
264 + self.spatial_merge_size = spatial_merge_size
265 + self.temporal_patch_size = temporal_patch_size
266 + self.out_hidden_size = out_hidden_size
267 + self.num_position_embeddings = num_position_embeddings
268 + self.deepstack_visual_indexes = (
269 + [8, 16, 24]
270 + if deepstack_visual_indexes is None
271 + else deepstack_visual_indexes
272 + )
273 + self.initializer_range = initializer_range
274 +
275 +
276 +class BailingMoeV3VLConfig(PretrainedConfig):
277 + model_type = "bailing_moe_v3_vl"
278 + keys_to_ignore_at_inference = ["past_key_values"]
279 + sub_configs = {
280 + "text_config": BailingMoeV3TextConfig,
281 + "vision_config": BailingMoeV3VisionConfig,
282 + }
283 +
284 + def __init__(
285 + self,
286 + *,
287 + text_config: BailingMoeV3TextConfig | dict[str, Any] | None = None,
288 + vision_config: BailingMoeV3VisionConfig | dict[str, Any] | None = None,
289 + image_token_id: int = 151655,
290 + video_token_id: int = 151656,
291 + vision_start_token_id: int = 151652,
292 + vision_end_token_id: int = 151653,
293 + tie_word_embeddings: bool = False,
294 + mrope_section: list[int] | tuple[int, ...] | None = None,
295 + architectures: list[str] | None = None,
296 + **kwargs: Any,
297 + ) -> None:
298 + kwargs.pop("model_type", None)
299 + if mrope_section is None:
300 + mrope_section = [12, 10, 10]
301 + self.mrope_section = list(mrope_section)
302 +
303 + if text_config is None:
304 + text_config = BailingMoeV3TextConfig()
305 + elif isinstance(text_config, dict):
306 + text_config = dict(text_config)
307 + nested_mrope_section = text_config.pop("mrope_section", None)
308 + if (
309 + nested_mrope_section is not None
310 + and list(nested_mrope_section) != self.mrope_section
311 + ):
312 + raise ValueError(
313 + "Bailing V3 VL has conflicting M-RoPE sections: "
314 + f"text={nested_mrope_section}, "
315 + f"top-level={self.mrope_section}"
316 + )
317 + text_config = BailingMoeV3TextConfig(**text_config)
318 + self.text_config = text_config
319 + _normalize_mrope_parameters(self.text_config, self.mrope_section)
320 +
321 + if vision_config is None:
322 + vision_config = BailingMoeV3VisionConfig()
323 + elif isinstance(vision_config, dict):
324 + vision_config = BailingMoeV3VisionConfig(**vision_config)
325 + self.vision_config = vision_config
326 +
327 + self.image_token_id = image_token_id
328 + self.video_token_id = video_token_id
329 + self.vision_start_token_id = vision_start_token_id
330 + self.vision_end_token_id = vision_end_token_id
331 + if architectures is None:
332 + architectures = ["BailingMoeV3VLForConditionalGeneration"]
333 +
334 + super().__init__(
335 + architectures=architectures,
336 + tie_word_embeddings=tie_word_embeddings,
337 + **kwargs,
338 + )
339 +
340 +
341 +__all__ = [
342 + "BailingMoeV3TextConfig",
343 + "BailingMoeV3VisionConfig",
344 + "BailingMoeV3VLConfig",
345 +]
vllm/v1/spec_decode/llm_base_proposer.py
+1
@@ -1356,6 +1356,7 @@ class SpecDecodeBaseProposer:
1356 # handle multimodality
1357 assert hasattr(target_model, "config")
1358 if self.get_model_name(target_model) in [
1359 + "BailingMoeV3VLForConditionalGeneration",
1360 "Cohere2VisionForConditionalGeneration",
1361 "Exaone4_5_ForConditionalGeneration",
1362 "GlmOcrForConditionalGeneration",