@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ba45bedfd1

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 9, 07:30 UTC

[model] Add support for openPangu7B-VL (#32449)

Signed-off-by: hujiaxin <524446785@qq.com> Signed-off-by: Emilie1001 <79921183+Emilie1001@users.noreply.github.com> Co-authored-by: Emilie1001 <79921183+Emilie1001@users.noreply.github.com>

hujiaxin0 committed Jan 30, 2026 at 15:54 UTC ba45bedfd15ab01af2be5ae28af03888f3683063

9 files changed +1601 -1

docs/models/supported_models.md

@@ -705,6 +705,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen

705 | `Molmo2ForConditionalGeneration` | Molmo2 | T + I+ / V | `allenai/Molmo2-4B`, `allenai/Molmo2-8B`, `allenai/Molmo2-O-7B` | ✅︎ | ✅︎ |

706 | `NVLM_D_Model` | NVLM-D 1.0 | T + I+ | `nvidia/NVLM-D-72B`, etc. | | ✅︎ |

707 | `OpenCUAForConditionalGeneration` | OpenCUA-7B | T + IE+ | `xlangai/OpenCUA-7B` | ✅︎ | ✅︎ |

708 +| `OpenPanguVLForConditionalGeneration` | openpangu-VL | T + IE+ + VE+ |`FreedomIntelligence/openPangu-VL-7B` | ✅︎ | ✅︎ |

709 | `Ovis` | Ovis2, Ovis1.6 | T + I+ | `AIDC-AI/Ovis2-1B`, `AIDC-AI/Ovis1.6-Llama3.2-3B`, etc. | | ✅︎ |

examples/offline_inference/vision_language.py

+32

@@ -1394,6 +1394,37 @@ def run_nvlm_d(questions: list[str], modality: str) -> ModelRequestData:

1394 )

1395

1396

1397 +# OpenPangu

1398 +def run_openpangu_vl(questions: list[str], modality: str) -> ModelRequestData:

1399 + model_name = "FreedomIntelligence/openPangu-VL-7B"

1400 +

1401 + engine_args = EngineArgs(

1402 + model=model_name,

1403 + max_model_len=4096,

1404 + max_num_seqs=4,

1405 + trust_remote_code=True,

1406 + enforce_eager=True,

1407 + limit_mm_per_prompt={modality: 1},

1408 + )

1409 +

1410 + if modality == "image":

1411 + placeholder = "[unused19]"

1412 + elif modality == "video":

1413 + placeholder = "[unused32]"

1414 +

1415 + prompts = [

1416 + (

1417 + f"<s>[unused9]系统：[unused10][unused9]用户：[unused18]{placeholder}[unused20]{question}[unused10][unused9]助手："

1418 + )

1419 + for question in questions

1420 + ]

1421 +

1422 + return ModelRequestData(

1423 + engine_args=engine_args,

1424 + prompts=prompts,

1425 + )

1426 +

1427 +

1428 # Ovis

1429 def run_ovis(questions: list[str], modality: str) -> ModelRequestData:

1430 assert modality == "image"

@@ -2051,6 +2082,7 @@ model_example_map = {

2082 "molmo2": run_molmo2,

2083 "nemotron_vl": run_nemotron_vl,

2084 "NVLM_D": run_nvlm_d,

2085 + "openpangu_vl": run_openpangu_vl,

2086 "ovis": run_ovis,

2087 "ovis2_5": run_ovis2_5,

2088 "paddleocr_vl": run_paddleocr_vl,

examples/offline_inference/vision_language_multi_image.py

+27

@@ -765,6 +765,32 @@ def load_nvlm_d(question: str, image_urls: list[str]) -> ModelRequestData:

765 )

766

767

768 +# OpenPangu

769 +def load_openpangu_vl(question: str, image_urls: list[str]) -> ModelRequestData:

770 + model_name = "FreedomIntelligence/openPangu-VL-7B"

771 +

772 + engine_args = EngineArgs(

773 + model=model_name,

774 + trust_remote_code=True,

775 + max_model_len=8192,

776 + max_num_seqs=2,

777 + enforce_eager=True,

778 + limit_mm_per_prompt={"image": len(image_urls)},

779 + )

780 +

781 + placeholders = "[unused18][unused19][unused20]" * len(image_urls)

782 + prompt = (

783 + f"<s>[unused9]系统：[unused10][unused9]用户：{question}{placeholders}"

784 + "[unused10][unused9]助手："

785 + )

786 +

787 + return ModelRequestData(

788 + engine_args=engine_args,

789 + prompt=prompt,

790 + image_data=[fetch_image(url) for url in image_urls],

791 + )

792 +

793 +

794 # Ovis

795 def load_ovis(question: str, image_urls: list[str]) -> ModelRequestData:

796 model_name = "AIDC-AI/Ovis2-1B"

@@ -1388,6 +1414,7 @@ model_example_map = {

1414 "mistral3": load_mistral3,

1415 "molmo2": load_molmo2,

1416 "NVLM_D": load_nvlm_d,

1417 + "openpangu_vl": load_openpangu_vl,

1418 "ovis": load_ovis,

1419 "ovis2_5": load_ovis2_5,

1420 "paddleocr_vl": load_paddleocr_vl,

tests/models/registry.py

@@ -873,6 +873,12 @@ _MULTIMODAL_EXAMPLE_MODELS = {

873 "OpenCUAForConditionalGeneration": _HfExamplesInfo(

874 "xlangai/OpenCUA-7B", trust_remote_code=True

875 ),

876 + "OpenPanguVLForConditionalGeneration": _HfExamplesInfo(

877 + "FreedomIntelligence/openPangu-VL-7B",

878 + trust_remote_code=True,

879 + max_model_len=4096,

880 + enforce_eager=True,

881 + ),

882 "Ovis": _HfExamplesInfo(

883 "AIDC-AI/Ovis2-1B",

884 trust_remote_code=True,

vllm/model_executor/layers/rotary_embedding/__init__.py

+16

@@ -15,6 +15,7 @@ from .linear_scaling_rope import LinearScalingRotaryEmbedding

15 from .llama3_rope import Llama3RotaryEmbedding

16 from .llama4_vision_rope import Llama4VisionRotaryEmbedding

17 from .mrope import MRotaryEmbedding

18 +from .mrope_interleaved import MRotaryEmbeddingInterleaved

19 from .ntk_scaling_rope import NTKScalingRotaryEmbedding

20 from .phi3_long_rope_scaled_rope import Phi3LongRoPEScaledRotaryEmbedding

21 from .xdrope import XDRotaryEmbedding

@@ -283,6 +284,21 @@ def get_rope(

284 long_factor,

285 **extra_kwargs,

286 )

287 + elif scaling_type == "openpangu":

288 + mrope_interleaved = rope_parameters.get("mrope_interleaved", False)

289 + if "mrope_section" in rope_parameters and mrope_interleaved:

290 + rotary_emb = MRotaryEmbeddingInterleaved(

291 + head_size,

292 + rotary_dim,

293 + max_position,

294 + base,

295 + is_neox_style,

296 + dtype,

297 + mrope_section=rope_parameters["mrope_section"],

298 + mrope_interleaved=mrope_interleaved,

299 + )

300 + else:

301 + raise ValueError("Pangu mrope lacks necessary parameters.")

302 else:

303 raise ValueError(f"Unknown RoPE scaling type {scaling_type}")

304 _ROPE_DICT[key] = rotary_emb

vllm/model_executor/layers/rotary_embedding/mrope_interleaved.py new

+185

@@ -0,0 +1,185 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +#

5 +# Adapted from vllm/model_executor/layers/rotary_embedding/__init__.py

7 +#

8 +# This file is a part of the vllm-ascend project.

9 +#

10 +# Licensed under the Apache License, Version 2.0 (the "License");

11 +# you may not use this file except in compliance with the License.

12 +# You may obtain a copy of the License at

13 +#

14 +# http://www.apache.org/licenses/LICENSE-2.0

15 +#

16 +# Unless required by applicable law or agreed to in writing, software

17 +# distributed under the License is distributed on an "AS IS" BASIS,

18 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

19 +# See the License for the specific language governing permissions and

20 +# limitations under the License.

21 +

22 +import torch

23 +

24 +from .mrope import MRotaryEmbedding

25 +

26 +

27 +# MRotaryEmbedding with interleaved

28 +class MRotaryEmbeddingInterleaved(MRotaryEmbedding):

29 + """Rotary Embedding with Multimodal Sections and Interleaved Support."""

30 +

31 + def __init__(

32 + self,

33 + head_size: int,

34 + rotary_dim: int,

35 + max_position_embeddings: int,

36 + base: float,

37 + is_neox_style: bool,

38 + dtype: torch.dtype,

39 + mrope_section: list[int],

40 + mrope_interleaved: bool = True,

41 + ) -> None:

42 + # Enlarge max_position_embeddings for video inputs

43 + self.cache_max_position_num = max_position_embeddings

44 + super().__init__(

45 + head_size,

46 + rotary_dim,

47 + self.cache_max_position_num,

48 + base,

49 + is_neox_style,

50 + dtype,

51 + )

52 +

53 + self.mrope_section = mrope_section

54 + self.mrope_interleaved = mrope_interleaved

55 +

56 + if self.mrope_section is None:

57 + raise ValueError("mrope_section cannot be None.")

58 + if sum(self.mrope_section) != rotary_dim // 2:

59 + raise ValueError("Sum of mrope_section must equal rotary_dim // 2.")

60 + if not self.mrope_interleaved:

61 + raise ValueError(

62 + "mrope_interleaved must be True when mrope_section is provided."

63 + )

64 +

65 + # Generate interleaved indices

66 + if len(mrope_section) == 2:

67 + h_num, w_num = mrope_section[0], mrope_section[1]

68 + mrope_dim = self.get_mrope_interleaved_id_list(h_num, w_num, 0)

69 + elif len(mrope_section) == 3:

70 + t_num, h_num, w_num = mrope_section[0], mrope_section[1], mrope_section[2]

71 + mrope_dim = self.get_mrope_interleaved_id_list(

72 + t_num, h_num, w_num, force_last=True

73 + )

74 + else:

75 + raise AssertionError(

76 + "Cannot support the length of mrope section is not 2 or 3."

77 + )

78 +

79 + mrope_dim = mrope_dim * 2

80 + self.mrope_dim = mrope_dim

81 +

82 + self.layer_cache = None

83 +

84 + def _rebuild_pos_emb(

85 + self,

86 + positions: torch.Tensor,

87 + ) -> tuple[torch.Tensor, torch.Tensor]:

88 + """Interleave the rotary embedding"""

89 + cos_sin = self.cos_sin_cache[positions]

90 + mrope_section_3d = [1] * len(self.mrope_dim)

91 + mrope_dim = self.mrope_dim

92 + cos_sin = torch.cat(

93 + [

94 + m[mrope_dim[i]]

95 + for i, m in enumerate(cos_sin.split(mrope_section_3d, dim=-1))

96 + ],

97 + dim=-1,

98 + )

99 + return cos_sin, torch.arange(cos_sin.shape[0], device=positions.device)

100 +

101 + def forward(

102 + self,

103 + positions: torch.Tensor,

104 + query: torch.Tensor,

105 + key: torch.Tensor | None = None,

106 + ) -> tuple[torch.Tensor, torch.Tensor | None]:

107 + """Forward pass with interleaved rotary embedding."""

108 + cos_sin, positions = self._rebuild_pos_emb(positions)

109 + cos, sin = cos_sin.chunk(2, dim=-1)

110 +

111 + query_shape = query.shape

112 + positions = positions.flatten()

113 + num_tokens = positions.shape[0]

114 + query = query.view(num_tokens, -1, self.head_size)

115 + query_rot = query[..., : self.rotary_dim]

116 + query_pass = query[..., self.rotary_dim :]

117 + query_rot = self.apply_rotary_emb.forward_native(

118 + query_rot,

119 + cos,

120 + sin,

121 + )

122 + query = torch.cat((query_rot, query_pass), dim=-1).reshape(query_shape)

123 +

124 + # key may be None in some cases, e.g. cross-layer KV sharing

125 + if key is not None:

126 + key_shape = key.shape

127 + key = key.view(num_tokens, -1, self.head_size)

128 + key_rot = key[..., : self.rotary_dim]

129 + key_pass = key[..., self.rotary_dim :]

130 + key_rot = self.apply_rotary_emb.forward_native(

131 + key_rot,

132 + cos,

133 + sin,

134 + )

135 + key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape)

136 + return query, key

137 +

138 + @staticmethod

139 + def get_mrope_interleaved_id_list(

140 + a: int, b: int, c: int, force_last: bool = False

141 + ) -> list[int]:

142 + """

143 + Generate an interleaved list of indices for multi-modal rotary embedding.

144 +

145 + Args:

146 + a: Number of indices for first modality

147 + b: Number of indices for second modality

148 + c: Number of indices for third modality

149 + force_last: Whether to force the last element to be from the first modality

150 +

151 + Returns:

152 + List of interleaved indices

153 + """

154 + if force_last:

155 + a -= 1

156 +

157 + counts = {0: a, 1: b, 2: c}

158 + placed = {k: 0 for k in counts}

159 + rem = counts.copy()

160 + seq: list[int] = []

161 + last = None

162 +

163 + total = a + b + c

164 + for _ in range(total):

165 + # Candidates: remaining > 0 and ≠ last

166 + cands = [k for k in rem if rem[k] > 0 and k != last]

167 + if not cands:

168 + # If only last remains, relax the condition

169 + cands = [k for k in rem if rem[k] > 0]

170 +

171 + # Select the rarest candidate

172 + try:

173 + best = min(cands, key=lambda k: (placed[k] / counts[k], k))

174 + except KeyError:

175 + best = 0

176 +

177 + seq.append(best)

178 + placed[best] += 1

179 + rem[best] -= 1

180 + last = best

181 +

182 + if force_last:

183 + seq.append(0)

184 +

185 + return seq

vllm/model_executor/models/openpangu.py

+7 -1

@@ -537,10 +537,16 @@ class OpenPanguEmbeddedAttention(nn.Module):

537 if is_gguf and config.model_type == "PanguEmbedded":

538 is_neox_style = False

539

540 + rope_parameters = config.rope_parameters or {}

541 + if rope_parameters is not None and rope_parameters.get(

542 + "mrope_interleaved", False

543 + ):

544 + rope_parameters["rope_type"] = "openpangu"

545 +

546 self.rotary_emb = get_rope(

547 self.head_dim,

548 max_position=self.max_position_embeddings,

543 - rope_parameters=config.rope_parameters,

549 + rope_parameters=rope_parameters,

550 is_neox_style=is_neox_style,

551 )

552

vllm/model_executor/models/openpangu_vl.py new

+1323

@@ -0,0 +1,1323 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +#

5 +# Adapted from vllm/model_executor/models/qwen2_5_vl.py

8 +#

9 +# This file is a part of the vllm-ascend project.

10 +#

11 +# Licensed under the Apache License, Version 2.0 (the "License");

12 +# you may not use this file except in compliance with the License.

13 +# You may obtain a copy of the License at

14 +#

15 +# http://www.apache.org/licenses/LICENSE-2.0

16 +#

17 +# Unless required by applicable law or agreed to in writing, software

18 +# distributed under the License is distributed on an "AS IS" BASIS,

19 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

20 +# See the License for the specific language governing permissions and

21 +# limitations under the License.

22 +

23 +from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence

24 +from functools import lru_cache, partial

25 +from typing import Annotated, Literal, Optional

26 +

27 +import torch

28 +import torch.nn as nn

29 +import torch.nn.functional as F

30 +from einops import rearrange

31 +from torchvision.transforms import v2

32 +from transformers.utils import logging

33 +

34 +from vllm.config import VllmConfig

35 +from vllm.distributed import parallel_state

36 +from vllm.distributed import utils as dist_utils

37 +from vllm.model_executor.layers.activation import _ACTIVATION_REGISTRY

38 +from vllm.model_executor.layers.attention.mm_encoder_attention import MMEncoderAttention

39 +from vllm.model_executor.layers.layernorm import RMSNorm

40 +from vllm.model_executor.layers.linear import (

41 + ColumnParallelLinear,

42 + MergedColumnParallelLinear,

43 + QKVParallelLinear,

44 + RowParallelLinear,

45 +)

46 +from vllm.model_executor.layers.quantization import QuantizationConfig

47 +from vllm.model_executor.layers.quantization.gptq import GPTQConfig

48 +from vllm.model_executor.layers.quantization.gptq_marlin import GPTQMarlinConfig

49 +from vllm.model_executor.layers.rotary_embedding.common import ApplyRotaryEmb

50 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader

51 +from vllm.model_executor.models.interfaces import (

52 + MultiModalEmbeddings,

53 + SupportsLoRA,

54 + SupportsMRoPE,

55 + SupportsMultiModal,

56 + SupportsPP,

57 +)

58 +from vllm.model_executor.models.module_mapping import MultiModelKeys

59 +from vllm.model_executor.models.qwen2_5_vl import (

60 + Qwen2_5_VLDummyInputsBuilder,

61 + Qwen2_5_VLMultiModalProcessor,

62 + Qwen2_5_VLProcessingInfo,

63 +)

64 +from vllm.model_executor.models.utils import (

65 + AutoWeightsLoader,

66 + WeightsMapper,

67 + init_vllm_registered_model,

68 + maybe_prefix,

69 +)

70 +from vllm.multimodal import MULTIMODAL_REGISTRY

71 +from vllm.multimodal.inputs import (

72 + MultiModalFeatureSpec,

73 + MultiModalKwargsItems,

74 +)

75 +from vllm.multimodal.parse import MultiModalDataItems

76 +from vllm.multimodal.processing import (

77 + PromptReplacement,

78 + PromptUpdate,

79 + PromptUpdateDetails,

80 +)

81 +from vllm.sequence import IntermediateTensors

82 +from vllm.utils.tensor_schema import TensorSchema, TensorShape

83 +from vllm.v1.attention.backends.registry import AttentionBackendEnum

84 +

85 +from .vision import get_vit_attn_backend

86 +

87 +logger = logging.get_logger(__name__)

88 +

89 +

90 +class OpenPanguVisionAttention(nn.Module):

91 + def __init__(

92 + self,

93 + embed_dim: int,

94 + num_heads: int,

95 + projection_size: int,

96 + quant_config: QuantizationConfig | None = None,

97 + prefix: str = "",

98 + ) -> None:

99 + super().__init__()

100 + self.hidden_size_per_attention_head = dist_utils.divide(

101 + projection_size, num_heads

102 + )

103 + self.tp_size = parallel_state.get_tensor_model_parallel_world_size()

104 + self.tp_rank = parallel_state.get_tensor_model_parallel_rank()

105 + self.num_attention_heads_per_partition = dist_utils.divide(

106 + num_heads, self.tp_size

107 + )

108 +

109 + self.qkv = QKVParallelLinear(

110 + hidden_size=embed_dim,

111 + head_size=self.hidden_size_per_attention_head,

112 + total_num_heads=num_heads,

113 + total_num_kv_heads=num_heads,

114 + bias=True,

115 + quant_config=quant_config,

116 + prefix=f"{prefix}.qkv",

117 + )

118 + self.proj = RowParallelLinear(

119 + input_size=projection_size,

120 + output_size=embed_dim,

121 + quant_config=quant_config,

122 + prefix=f"{prefix}.proj",

123 + )

124 + self.attn = MMEncoderAttention(

125 + num_heads=self.num_attention_heads_per_partition,

126 + head_size=self.hidden_size_per_attention_head,

127 + scale=self.hidden_size_per_attention_head**-0.5,

128 + )

129 + self.apply_rotary_emb = ApplyRotaryEmb(enforce_enable=True)

130 +

131 + def forward(

132 + self,

133 + x: torch.Tensor,

134 + cu_seqlens: torch.Tensor,

135 + cos: torch.Tensor,

136 + sin: torch.Tensor,

137 + ) -> torch.Tensor:

138 + seq_length, _ = x.size()

139 + x, bias = self.qkv(x)

140 + if bias is not None:

141 + x = x + bias

142 + q, k, v = x.chunk(3, dim=1)

143 +

144 + q, k, v = (

145 + rearrange(

146 + x, "s (b n d) -> b s n d", d=self.hidden_size_per_attention_head, b=1

147 + ).contiguous()

148 + for x in (q, k, v)

149 + )

150 + qk_concat = torch.cat([q, k], dim=0)

151 + qk_rotated = self.apply_rotary_emb(qk_concat, cos, sin)

152 + q, k = torch.chunk(qk_rotated, 2, dim=0)

153 +

154 + max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max()

155 + context_layer = self.attn(

156 + query=q,

157 + key=k,

158 + value=v,

159 + cu_seqlens=cu_seqlens,

160 + max_seqlen=max_seqlen,

161 + )

162 + context_layer = rearrange(

163 + context_layer, "b s h d -> s (b h d)", b=1

164 + ).contiguous()

165 + output, bias = self.proj(context_layer)

166 + if bias is not None:

167 + output = output + bias

168 + return output

169 +

170 +

171 +class OpenPanguVisionMLP(nn.Module):

172 + def __init__(

173 + self,

174 + in_features: int,

175 + hidden_features: int,

176 + bias: bool = False,

177 + act_fn: Callable[[torch.Tensor], torch.Tensor] = F.silu,

178 + vision_config=None,

179 + quant_config: QuantizationConfig | None = None,

180 + prefix: str = "",

181 + ):

182 + super().__init__()

183 + self.hidden_act = vision_config.hidden_act

184 + if self.hidden_act == "silu":

185 + tp_size = parallel_state.get_tensor_model_parallel_world_size()

186 + if hidden_features % tp_size != 0:

187 + hidden_features = (hidden_features + tp_size - 1) // tp_size * tp_size

188 + self.gate_up_proj = MergedColumnParallelLinear(

189 + input_size=in_features,

190 + output_sizes=[hidden_features] * 2,

191 + bias=bias,

192 + quant_config=quant_config,

193 + prefix=f"{prefix}.gate_up_proj",

194 + )

195 + else:

196 + self.up_proj = ColumnParallelLinear(

197 + in_features,

198 + hidden_features,

199 + bias=bias,

200 + quant_config=quant_config,

201 + prefix=f"{prefix}.up_proj",

202 + )

203 +

204 + self.down_proj = RowParallelLinear(

205 + hidden_features,

206 + in_features,

207 + bias=bias,

208 + quant_config=quant_config,

209 + prefix=f"{prefix}.down_proj",

210 + )

211 + self.act_fn = act_fn

212 +

213 + def forward(self, x: torch.Tensor):

214 + if self.hidden_act == "silu":

215 + x, _ = self.gate_up_proj(x)

216 + else:

217 + x, _ = self.up_proj(x)

218 + x = self.act_fn(x)

219 + x, _ = self.down_proj(x)

220 + return x

221 +

222 +

223 +class OpenPanguVisionBlock(nn.Module):

224 + def __init__(

225 + self,

226 + dim: int,

227 + num_heads: int,

228 + mlp_hidden_dim: int,

229 + act_fn: Callable[[torch.Tensor], torch.Tensor] = F.silu,

230 + norm_layer: Callable[[int], nn.Module] | None = None,

231 + vision_config=None,

232 + quant_config: QuantizationConfig | None = None,

233 + prefix: str = "",

234 + ) -> None:

235 + super().__init__()

236 + if norm_layer is None:

237 + norm_layer = partial(nn.LayerNorm, eps=1e-6)

238 + self.norm1 = norm_layer(dim)

239 + self.norm2 = norm_layer(dim)

240 + self.attn = OpenPanguVisionAttention(

241 + embed_dim=dim,

242 + num_heads=num_heads,

243 + projection_size=dim,

244 + quant_config=quant_config,

245 + prefix=f"{prefix}.attn",

246 + )

247 + self.mlp = OpenPanguVisionMLP(

248 + dim,

249 + mlp_hidden_dim,

250 + act_fn=act_fn,

251 + bias=True,

252 + vision_config=vision_config,

253 + quant_config=quant_config,

254 + prefix=f"{prefix}.mlp",

255 + )

256 +

257 + def forward(

258 + self,

259 + hidden_states: torch.Tensor,

260 + cu_seqlens: torch.Tensor,

261 + cos: torch.Tensor,

262 + sin: torch.Tensor,

263 + ) -> torch.Tensor:

264 + hidden_states = hidden_states + self.attn(

265 + self.norm1(hidden_states), cu_seqlens=cu_seqlens, cos=cos, sin=sin

266 + )

267 + hidden_states = hidden_states + self.mlp(self.norm2(hidden_states))

268 + return hidden_states

269 +

270 +

271 +class OpenPanguVisionRotaryEmbedding(nn.Module):

272 + def __init__(self, dim: int, theta: float = 10000.0) -> None:

273 + super().__init__()

274 + self.inv_freq = 1.0 / (

275 + theta ** (torch.arange(0, dim, 2, dtype=torch.float) / dim)

276 + )

277 + self._seq_len_cached = 0

278 + self._freqs_cached = None

279 +

280 + def update_freqs_cache(self, seqlen: int) -> None:

281 + if seqlen > self._seq_len_cached:

282 + seqlen *= 2

283 + self._seq_len_cached = seqlen

284 + seq = torch.arange(

285 + seqlen, device=self.inv_freq.device, dtype=self.inv_freq.dtype

286 + )

287 + freqs = torch.outer(seq, self.inv_freq)

288 + self._freqs_cached = freqs

289 +

290 + def forward(self, seqlen: int) -> torch.Tensor:

291 + self.update_freqs_cache(seqlen)

292 + return (

293 + self._freqs_cached[:seqlen]

294 + if self._freqs_cached is not None

295 + else self._freqs_cached

296 + )

297 +

298 +

299 +class OpenPanguVisionPatchEmbed(nn.Module):

300 + def __init__(

301 + self,

302 + patch_size: int = 14,

303 + temporal_patch_size: int = 2,

304 + in_channels: int = 3,

305 + hidden_size: int = 1152,

306 + ) -> None:

307 + super().__init__()

308 + self.patch_size = patch_size

309 + self.temporal_patch_size = temporal_patch_size

310 + self.hidden_size = hidden_size

311 + self.input_size = (

312 + self.patch_size * self.patch_size * in_channels * self.temporal_patch_size

313 + )

314 +

315 + kernel_size = (temporal_patch_size, patch_size, patch_size)

316 + self.proj = nn.Conv3d(

317 + in_channels,

318 + hidden_size,

319 + kernel_size=kernel_size,

320 + stride=kernel_size,

321 + bias=False,

322 + )

323 +

324 + def forward(self, x: torch.Tensor) -> torch.Tensor:

325 + if x.shape[-1] != self.input_size:

326 + x = torch.cat(

327 + [

328 + x.reshape(-1, self.patch_size * self.patch_size),

329 + x.reshape(-1, self.patch_size * self.patch_size),

330 + ],

331 + dim=-1,

332 + ).reshape(-1, self.input_size)

333 + x = x.matmul(self.proj.weight.data.view(self.hidden_size, -1).transpose(0, 1))

334 + return x

335 +

336 +

337 +class OpenPanguVisionPatchMerger(nn.Module):

338 + def __init__(

339 + self,

340 + d_model: int,

341 + context_dim: int,

342 + norm_layer: Callable[[int], nn.Module] | None = None,

343 + spatial_merge_size: int = 2,

344 + quant_config: QuantizationConfig | None = None,

345 + prefix: str = "",

346 + ) -> None:

347 + super().__init__()

348 + if norm_layer is None:

349 + norm_layer = partial(nn.LayerNorm, eps=1e-6)

350 + self.hidden_size = context_dim * (spatial_merge_size**2)

351 + self.ln_q = norm_layer(context_dim)

352 + self.mlp = nn.Sequential(

353 + ColumnParallelLinear(

354 + self.hidden_size,

355 + self.hidden_size,

356 + bias=True,

357 + quant_config=quant_config,

358 + prefix=f"{prefix}.mlp.0",

359 + return_bias=False,

360 + ),

361 + nn.GELU(),

362 + RowParallelLinear(

363 + self.hidden_size,

364 + d_model,

365 + bias=True,

366 + quant_config=quant_config,

367 + prefix=f"{prefix}.mlp.2",

368 + return_bias=False,

369 + ),

370 + )

371 +

372 + def forward(self, x: torch.Tensor) -> torch.Tensor:

373 + return self.mlp(self.ln_q(x).view(-1, self.hidden_size))

374 +

375 +

376 +class OpenPanguVisionTransformer(nn.Module):

377 + def __init__(

378 + self,

379 + vision_config,

380 + out_hidden_size,

381 + hidden_size,

382 + norm_eps: float = 1e-6,

383 + quant_config: QuantizationConfig | None = None,

384 + prefix: str = "",

385 + interleaved=False,

386 + ) -> None:

387 + super().__init__()

388 + self.hidden_size = vision_config.hidden_size

389 + self.num_heads = vision_config.num_heads

390 + self.window_size = vision_config.window_size

391 + self.patch_size = vision_config.patch_size

392 + self.spatial_merge_size = vision_config.spatial_merge_size

393 + self.fullatt_block_indexes = vision_config.fullatt_block_indexes

394 + self.spatial_merge_unit = self.spatial_merge_size**2

395 +

396 + norm_layer = partial(RMSNorm, eps=norm_eps)

397 + self.interleaved = interleaved

398 + self.out_hidden_size = vision_config.out_hidden_size

399 + self.hidden_act = vision_config.hidden_act

400 +

401 + head_dim = self.hidden_size // self.num_heads

402 + self.attn_backend = get_vit_attn_backend(

403 + head_size=head_dim,

404 + dtype=torch.get_default_dtype(),

405 + )

406 +

407 + if self.attn_backend not in {

408 + AttentionBackendEnum.FLASH_ATTN,

409 + }:

410 + raise RuntimeError(

411 + f"Pangu-VL does not support {self.attn_backend} backend now."

412 + )

413 + self.rotary_pos_emb = OpenPanguVisionRotaryEmbedding(head_dim // 2)

414 + self.patch_embed = OpenPanguVisionPatchEmbed(

415 + patch_size=vision_config.patch_size,

416 + temporal_patch_size=vision_config.temporal_patch_size,

417 + in_channels=vision_config.in_channels,

418 + hidden_size=self.hidden_size,

419 + )

420 + self.blocks = nn.ModuleList(

421 + [

422 + OpenPanguVisionBlock(

423 + dim=self.hidden_size,

424 + num_heads=self.num_heads,

425 + mlp_hidden_dim=vision_config.intermediate_size,

426 + act_fn=_ACTIVATION_REGISTRY[vision_config.hidden_act],

427 + vision_config=vision_config,

428 + norm_layer=norm_layer,

429 + quant_config=quant_config,

430 + prefix=f"{prefix}.blocks.{layer_idx}",

431 + )

432 + for layer_idx in range(vision_config.depth)

433 + ]

434 + )

435 + self.tp_size = parallel_state.get_tensor_model_parallel_world_size()

436 + self.tp_rank = parallel_state.get_tensor_model_parallel_rank()

437 + self.hidden_size_per_attention_head = dist_utils.divide(

438 + self.hidden_size, self.num_heads

439 + )

440 +

441 + self.select_layer = getattr(

442 + vision_config, "mm_unit_vision_select_layer", [-1, -3]

443 + )

444 + self.select_index = [vision_config.depth + i for i in self.select_layer]

445 + self.select_index = self.select_index[::-1]

446 + self.select_layer = [-1 * (i + 1) for i in range(len(self.select_index))]

447 +

448 + self.take_indices = self.select_index

449 +

450 + self.final_layernorm = RMSNorm(self.hidden_size, eps=norm_eps)

451 + self.merger = nn.ModuleList(

452 + [

453 + OpenPanguVisionPatchMerger(

454 + d_model=vision_config.out_hidden_size,

455 + context_dim=self.hidden_size,

456 + norm_layer=norm_layer,

457 + spatial_merge_size=self.spatial_merge_size,

458 + quant_config=quant_config,

459 + prefix=f"{prefix}.merger.{i}",

460 + )

461 + for i in range(len(self.select_layer))

462 + ]

463 + )

464 + self.vision_projection = ProjectionSingle(out_hidden_size, hidden_size)

465 +

466 + @property

467 + def dtype(self) -> torch.dtype:

468 + return self.patch_embed.proj.weight.dtype

469 +

470 + @property

471 + def device(self) -> torch.device:

472 + return self.patch_embed.proj.weight.device

473 +

474 + def cal_cos_sin(self, rotary_pos_emb):

475 + cos = rotary_pos_emb.cos()

476 + sin = rotary_pos_emb.sin()

477 + return cos, sin

478 +

479 + def rot_pos_emb(self, grid_thw: torch.Tensor) -> torch.Tensor:

480 + # see https://github.com/huggingface/transformers/blob/main/src/transformers/models/qwen2_5_vl/modular_qwen2_5_vl.py for details. #L209 # noqa: E501

481 + pos_ids = []

482 + for t, h, w in grid_thw:

483 + hpos_ids = torch.arange(h).unsqueeze(1).expand(-1, w)

484 + wpos_ids = torch.arange(w).unsqueeze(0).expand(h, -1)

485 + hpos_ids = (

486 + hpos_ids.reshape(

487 + h // self.spatial_merge_size,

488 + self.spatial_merge_size,

489 + w // self.spatial_merge_size,

490 + self.spatial_merge_size,

491 + )

492 + .permute(0, 2, 1, 3)

493 + .flatten()

494 + )

495 + wpos_ids = (

496 + wpos_ids.reshape(

497 + h // self.spatial_merge_size,

498 + self.spatial_merge_size,

499 + w // self.spatial_merge_size,

500 + self.spatial_merge_size,

501 + )

502 + .permute(0, 2, 1, 3)

503 + .flatten()

504 + )

505 + pos_ids.append(torch.stack([hpos_ids, wpos_ids], dim=-1).repeat(t, 1))

506 + pos_ids = torch.cat(pos_ids, dim=0)

507 + max_grid_size = grid_thw[:, 1:].max()

508 + rotary_pos_emb_full = self.rotary_pos_emb(max_grid_size)

509 + rotary_pos_emb = rotary_pos_emb_full[pos_ids].flatten(1)

510 + return rotary_pos_emb

511 +

512 + def get_window_index(self, grid_thw):

513 + # see https://github.com/huggingface/transformers/blob/main/src/transformers/models/qwen2_5_vl/modular_qwen2_5_vl.py for details. #L238 # noqa: E501

514 + window_index: list = []

515 + cu_window_seqlens: list = [0]

516 + window_index_id = 0

517 + vit_merger_window_size = (

518 + self.window_size // self.spatial_merge_size // self.patch_size

519 + )

520 +

521 + for grid_t, grid_h, grid_w in grid_thw:

522 + llm_grid_h = grid_h // self.spatial_merge_size

523 + llm_grid_w = grid_w // self.spatial_merge_size

524 + index = torch.arange(grid_t * llm_grid_h * llm_grid_w).reshape(

525 + grid_t, llm_grid_h, llm_grid_w

526 + )

527 + pad_h = vit_merger_window_size - llm_grid_h % vit_merger_window_size

528 + pad_w = vit_merger_window_size - llm_grid_w % vit_merger_window_size

529 + num_windows_h = (llm_grid_h + pad_h) // vit_merger_window_size

530 + num_windows_w = (llm_grid_w + pad_w) // vit_merger_window_size

531 + index_padded = F.pad(index, (0, pad_w, 0, pad_h), "constant", -100)

532 + index_padded = index_padded.reshape(

533 + grid_t,

534 + num_windows_h,

535 + vit_merger_window_size,

536 + num_windows_w,

537 + vit_merger_window_size,

538 + )

539 + index_padded = index_padded.permute(0, 1, 3, 2, 4).reshape(

540 + grid_t,

541 + num_windows_h * num_windows_w,

542 + vit_merger_window_size,

543 + vit_merger_window_size,

544 + )

545 + seqlens = (index_padded != -100).sum([2, 3]).reshape(-1)

546 + index_padded = index_padded.reshape(-1)

547 + index_new = index_padded[index_padded != -100]

548 + window_index.append(index_new + window_index_id)

549 + cu_seqlens_tmp = (

550 + seqlens.cumsum(0) * self.spatial_merge_unit + cu_window_seqlens[-1]

551 + )

552 + cu_window_seqlens.extend(cu_seqlens_tmp.tolist())

553 + window_index_id += (grid_t * llm_grid_h * llm_grid_w).item()

554 + window_index = torch.cat(window_index, dim=0)

555 + return window_index, cu_window_seqlens

556 +

557 + def forward(

558 + self,

559 + x: torch.Tensor,

560 + grid_thw: torch.Tensor,

561 + ) -> torch.Tensor:

562 + # compute cu_seqlens

563 + cu_seqlens = (

564 + torch.repeat_interleave(grid_thw[:, 1] * grid_thw[:, 2], grid_thw[:, 0])

565 + .to(torch.int32)

566 + .to(x.device)

567 + )

568 + cu_seqlens = torch.cumsum(cu_seqlens, dim=0, dtype=torch.int32)

569 + cu_seqlens = F.pad(cu_seqlens, (1, 0), "constant", 0)

570 +

571 + x = self.patch_embed(x)

572 +

573 + rotary_pos_emb = self.rot_pos_emb(grid_thw)

574 +

575 + window_index, cu_window_seqlens = self.get_window_index(grid_thw)

576 + cu_window_seqlens = torch.tensor(

577 + cu_window_seqlens,

578 + device=x.device,

579 + dtype=grid_thw.dtype if torch.jit.is_tracing() else torch.int32,

580 + )

581 + cu_window_seqlens = torch.unique_consecutive(cu_window_seqlens)

582 + seq_len, _ = x.size()

583 + x = x.reshape(seq_len // self.spatial_merge_unit, self.spatial_merge_unit, -1)

584 + x = x[window_index, :, :]

585 + x = x.reshape(seq_len, -1)

586 + rotary_pos_emb = rotary_pos_emb.reshape(

587 + seq_len // self.spatial_merge_unit, self.spatial_merge_unit, -1

588 + )

589 + rotary_pos_emb = rotary_pos_emb[window_index, :, :]

590 + rotary_pos_emb = rotary_pos_emb.reshape(seq_len, -1)

591 +

592 + cos, sin = self.cal_cos_sin(rotary_pos_emb.to(x.dtype))

593 +

594 + intermediates = []

595 + for layer_num, blk in enumerate(self.blocks):

596 + if layer_num in self.fullatt_block_indexes:

597 + cu_seqlens_now = cu_seqlens

598 + else:

599 + cu_seqlens_now = cu_window_seqlens

600 + x = blk(x, cu_seqlens=cu_seqlens_now, cos=cos, sin=sin)

601 + if layer_num in self.take_indices:

602 + ln_hs = self.final_layernorm(x)

603 + intermediates.append(ln_hs)

604 +

605 + image_embeddings_list = []

606 + for idx, sl in enumerate(self.select_layer):

607 + image_embeddings_list.append(self.merger[idx](intermediates[sl]))

608 + x = sum(image_embeddings_list)

609 +

610 + reverse_indices = torch.argsort(window_index)

611 + x = x[reverse_indices, :]

612 + x = self.vision_projection(x)

613 + return x

614 +

615 + def load_weights(self, weights) -> set[str]:

616 + def _padding_weight(name: str, w: torch.Tensor) -> torch.Tensor:

617 + if "gate_proj" in name or "up_proj" in name:

618 + dim, size = 0, w.size(0)

619 + elif "down_proj" in name:

620 + dim, size = 1, w.size(-1)

621 + else:

622 + return w

623 + pad_len = -size % self.tp_size

624 + if pad_len == 0:

625 + return w

626 + pad = [0] * (w.ndim * 2)

627 + pad[-(dim + 1) * 2 + 1] = pad_len

628 + return F.pad(w, pad, mode="constant", value=0)

629 +

630 + stacked_params_mapping = [

631 + ("attn.qkv.", "attn.q.", "q"),

632 + ("attn.qkv.", "attn.k.", "k"),

633 + ("attn.qkv.", "attn.v.", "v"),

634 + ]

635 + if self.hidden_act == "silu":

636 + stacked_params_mapping.extend(

637 + [

638 + ("gate_up_proj", "gate_proj", 0),

639 + ("gate_up_proj", "up_proj", 1),

640 + ]

641 + )

642 + params_dict = dict(self.named_parameters(remove_duplicate=False))

643 + loaded_params: set[str] = set()

644 +

645 + for name, loaded_weight in weights:

646 + if self.hidden_act == "silu":

647 + loaded_weight = _padding_weight(name, loaded_weight)

648 + for param_name, weight_name, shard_id in stacked_params_mapping:

649 + if weight_name not in name:

650 + continue

651 + name = name.replace(weight_name, param_name)

652 +

653 + param = params_dict[name]

654 + weight_loader = param.weight_loader

655 + weight_loader(param, loaded_weight, shard_id)

656 + break

657 + else:

658 + param = params_dict[name]

659 + weight_loader = getattr(param, "weight_loader", default_weight_loader)

660 + weight_loader(param, loaded_weight)

661 + loaded_params.add(name)

662 + return loaded_params

663 +

664 +

665 +class ProjectionSingle(nn.Module):

666 + def __init__(self, i_hidden_size: int, t_hidden_size: int):

667 + super().__init__()

668 + self.act = F.silu

669 + self.fc1 = nn.Linear(i_hidden_size, t_hidden_size, bias=True)

670 +

671 + def forward(self, hidden_states):

672 + x = self.act(hidden_states)

673 + return self.fc1(x)

674 +

675 +

676 +class OpenPanguVLProcessingInfo(Qwen2_5_VLProcessingInfo):

677 + def get_hf_config(self):

678 + return self.ctx.model_config.hf_config

679 +

680 + def get_hf_processor(

681 + self,

682 + *,

683 + min_pixels: int | None = None,

684 + max_pixels: int | None = None,

685 + size: dict[str, int] | None = None,

686 + fps: float | list[float] | None = None,

687 + **kwargs: object,

688 + ):

689 + if fps is not None:

690 + kwargs["fps"] = fps

691 +

692 + return self.ctx.get_hf_processor(

693 + use_fast=kwargs.pop("use_fast", True),

694 + **kwargs,

695 + )

696 +

697 +

698 +class OpenPanguVLImagePixelInputs(TensorSchema):

699 + type: Literal["pixel_values"]

700 +

701 + pixel_values: Annotated[

702 + torch.Tensor,

703 + TensorShape("np", "cps"),

704 + ]

705 + image_grid_thw: Annotated[

706 + torch.Tensor,

707 + TensorShape("ni", 3),

708 + ]

709 +

710 +

711 +class OpenPanguVLImageEmbeddingInputs(TensorSchema):

712 + type: Literal["image_embeds"]

713 +

714 + image_embeds: Annotated[

715 + torch.Tensor,

716 + TensorShape("nf", "hs"),

717 + ]

718 + image_grid_thw: Annotated[

719 + torch.Tensor,

720 + TensorShape("ni", 3),

721 + ]

722 +

723 +

724 +class OpenPanguVLVideoPixelInputs(TensorSchema):

725 + type: Literal["pixel_values_videos"]

726 +

727 + pixel_values_videos: Annotated[

728 + torch.Tensor,

729 + TensorShape("np", "ctps"),

730 + ]

731 + video_grid_thw: Annotated[

732 + torch.Tensor,

733 + TensorShape("nv", 3),

734 + ]

735 +

736 +

737 +class OpenPanguVLVideoEmbeddingInputs(TensorSchema):

738 + type: Literal["video_embeds"]

739 +

740 + video_embeds: Annotated[

741 + torch.Tensor,

742 + TensorShape("nf", "hs"),

743 + ]

744 + video_grid_thw: Annotated[

745 + torch.Tensor,

746 + TensorShape("nv", 3),

747 + ]

748 +

749 +

750 +class OpenPanguVLMultiModalProcessor(Qwen2_5_VLMultiModalProcessor):

751 + def _get_prompt_updates(

752 + self,

753 + mm_items: MultiModalDataItems,

754 + hf_processor_mm_kwargs: Mapping[str, any],

755 + out_mm_kwargs: MultiModalKwargsItems,

756 + ) -> Sequence[PromptUpdate]:

757 + hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)

758 + image_processor = self.info.get_image_processor(**hf_processor_mm_kwargs)

759 + tokenizer = self.info.get_tokenizer()

760 + vocab = tokenizer.get_vocab()

761 + image_token = hf_processor.image_token

762 + video_token = hf_processor.video_token

763 + vision_start_token = hf_processor.vision_start_token

764 + vision_end_token = hf_processor.vision_end_token

765 + image_token_id = vocab[image_token]

766 + video_token_id = vocab[video_token]

767 + vision_start_token_id = vocab[vision_start_token]

768 + vision_end_token_id = vocab[vision_end_token]

769 + placeholder = {

770 + "image": image_token_id,

771 + "video": video_token_id,

772 + }

773 +

774 + merge_length = image_processor.merge_size**2

775 +

776 + def get_replacement_openpangu_vision(item_idx: int, modality: str):

777 + out_item = out_mm_kwargs[modality][item_idx]

778 + grid_thw = out_item[f"{modality}_grid_thw"].data

779 + if not isinstance(grid_thw, torch.Tensor):

780 + raise TypeError("Expected 'grid_thw' to be a Tensor")

781 + if modality == "image":

782 + image_token_id_total = [image_token_id] * (

783 + int(grid_thw.prod()) // merge_length

784 + )

785 + return image_token_id_total

786 + else:

787 + # When modality is video

788 + grid_t, grid_h, grid_w = grid_thw

789 + video_seq_length_per_time = (grid_h * grid_w).item() // merge_length

790 + video_token_id_per_time = (

791 + [vision_start_token_id]

792 + + [video_token_id] * video_seq_length_per_time

793 + + [vision_end_token_id]

794 + )

795 + video_token_id_total = video_token_id_per_time * grid_t.item()

796 + video_token_id_middle = video_token_id_total[1:-1]

797 + return PromptUpdateDetails.select_token_id(

798 + video_token_id_middle,

799 + embed_token_id=video_token_id,

800 + )

801 +

802 + return [

803 + PromptReplacement(

804 + modality=modality,

805 + target=[placeholder[modality]],

806 + replacement=partial(

807 + get_replacement_openpangu_vision, modality=modality

808 + ),

809 + )

810 + for modality in ("image", "video")

811 + ]

812 +

813 +

814 +class OpenPanguVLDummyInputsBuilder(Qwen2_5_VLDummyInputsBuilder):

815 + pass

816 +

817 +

818 +@MULTIMODAL_REGISTRY.register_processor(

819 + OpenPanguVLMultiModalProcessor,

820 + info=OpenPanguVLProcessingInfo,

821 + dummy_inputs=OpenPanguVLDummyInputsBuilder,

822 +)

823 +class OpenPanguVLForConditionalGeneration(

824 + nn.Module, SupportsMultiModal, SupportsLoRA, SupportsPP, SupportsMRoPE

825 +):

826 + hf_to_vllm_mapper = WeightsMapper(

827 + orig_to_new_prefix={

828 + "model.language_model.": "language_model.model.",

829 + "model.visual.": "visual.",

830 + "lm_head.": "language_model.lm_head.",

831 + "model.": "language_model.model.",

832 + }

833 + )

834 + packed_modules_mapping = {

835 + "qkv_proj": ["q_proj", "k_proj", "v_proj"],

836 + "gate_up_proj": ["gate_proj", "up_proj"],

837 + }

838 +

839 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):

840 + super().__init__()

841 + config = vllm_config.model_config.hf_config

842 + self.config = config

843 + self.vllm_config = vllm_config

844 + quant_config = vllm_config.quant_config

845 + self.visual = OpenPanguVisionTransformer(

846 + vision_config=config.vision_config,

847 + out_hidden_size=config.vision_config.out_hidden_size,

848 + hidden_size=config.hidden_size,

849 + norm_eps=getattr(config.vision_config, "rms_norm_eps", 1e-6),

850 + quant_config=self._maybe_ignore_quant_config(quant_config),

851 + prefix=maybe_prefix(prefix, "visual"),

852 + )

853 +

854 + self.language_model = init_vllm_registered_model(

855 + vllm_config=vllm_config,

856 + prefix=maybe_prefix("openpangu", "language_model"),

857 + architectures=["PanguEmbeddedForCausalLM"],

858 + )

859 + self.make_empty_intermediate_tensors = (

860 + self.language_model.make_empty_intermediate_tensors

861 + )

862 + self._parse_preprocess_params(config.vision_config)

863 +

864 + def _parse_preprocess_params(self, vision_config):

865 + self.channel = vision_config.in_channels

866 + self.patch_size = vision_config.patch_size

867 + from vllm.multimodal import MULTIMODAL_REGISTRY

868 +

869 + image_processor = (

870 + MULTIMODAL_REGISTRY.create_processor(self.vllm_config.model_config)

871 + .info.get_hf_processor()

872 + .image_processor

873 + )

874 + self.do_rescale = image_processor.do_rescale

875 + self.rescale_factor = image_processor.rescale_factor

876 + self.do_normalize = image_processor.do_normalize

877 + self.image_mean = tuple(image_processor.image_mean)

878 + self.image_std = tuple(image_processor.image_std)

879 +

880 + def _maybe_ignore_quant_config(self, quant_config: QuantizationConfig):

881 + if isinstance(quant_config, (GPTQConfig, GPTQMarlinConfig)):

882 + return None

883 + return quant_config

884 +

885 + def _validate_and_reshape_mm_tensor(

886 + self, mm_input: object, name: str

887 + ) -> torch.Tensor:

888 + if not isinstance(mm_input, (torch.Tensor, list)):

889 + raise ValueError(f"Incorrect type of {name}. Got type: {type(mm_input)}")

890 + if isinstance(mm_input, torch.Tensor):

891 + if mm_input.ndim == 2:

892 + return mm_input

893 + if mm_input.ndim != 3:

894 + raise ValueError(

895 + f"{name} should be 2D or batched 3D tensor. "

896 + f"Got ndim: {mm_input.ndim} "

897 + f"(shape={mm_input.shape})"

898 + )

899 + return torch.concat(list(mm_input))

900 + else:

901 + return torch.concat(mm_input)

902 +

903 + def _parse_and_validate_image_input(self, **kwargs: object):

904 + pixel_values = kwargs.pop("pixel_values", None)

905 + image_embeds = kwargs.pop("image_embeds", None)

906 + image_grid_thw = kwargs.pop("image_grid_thw", None)

907 +

908 + if pixel_values is None and image_embeds is None:

909 + return None

910 +

911 + if pixel_values is not None:

912 + pixel_values = self._validate_and_reshape_mm_tensor(

913 + pixel_values, "image pixel values"

914 + )

915 + image_grid_thw = self._validate_and_reshape_mm_tensor(

916 + image_grid_thw, "image grid_thw"

917 + )

918 +

919 + if not isinstance(pixel_values, (torch.Tensor, list)):

920 + raise ValueError(

921 + "Incorrect type of image pixel values. "

922 + f"Got type: {type(pixel_values)}"

923 + )

924 +

925 + return OpenPanguVLImagePixelInputs(

926 + type="pixel_values",

927 + pixel_values=pixel_values,

928 + image_grid_thw=image_grid_thw,

929 + )

930 +

931 + if image_embeds is not None:

932 + image_embeds = self._validate_and_reshape_mm_tensor(

933 + image_embeds, "image embeds"

934 + )

935 + image_grid_thw = self._validate_and_reshape_mm_tensor(

936 + image_grid_thw, "image grid_thw"

937 + )

938 +

939 + if not isinstance(image_embeds, torch.Tensor):

940 + raise ValueError(

941 + "Incorrect type of image embeddings. "

942 + f"Got type: {type(image_embeds)}"

943 + )

944 + return OpenPanguVLImageEmbeddingInputs(

945 + type="image_embeds",

946 + image_embeds=image_embeds,

947 + image_grid_thw=image_grid_thw,

948 + )

949 +

950 + def _parse_and_validate_video_input(self, **kwargs: object):

951 + pixel_values_videos = kwargs.pop("pixel_values_videos", None)

952 + video_embeds = kwargs.pop("video_embeds", None)

953 + video_grid_thw = kwargs.pop("video_grid_thw", None)

954 +

955 + if pixel_values_videos is None and video_embeds is None:

956 + return None

957 +

958 + if pixel_values_videos is not None:

959 + pixel_values_videos = self._validate_and_reshape_mm_tensor(

960 + pixel_values_videos, "video pixel values"

961 + )

962 + video_grid_thw = self._validate_and_reshape_mm_tensor(

963 + video_grid_thw, "video grid_thw"

964 + )

965 +

966 + return OpenPanguVLVideoPixelInputs(

967 + type="pixel_values_videos",

968 + pixel_values_videos=pixel_values_videos,

969 + video_grid_thw=video_grid_thw,

970 + )

971 +

972 + if video_embeds is not None:

973 + video_embeds = self._validate_and_reshape_mm_tensor(

974 + video_embeds, "video embeds"

975 + )

976 + video_grid_thw = self._validate_and_reshape_mm_tensor(

977 + video_grid_thw, "video grid_thw"

978 + )

979 +

980 + if not isinstance(video_embeds, torch.Tensor):

981 + raise ValueError(

982 + "Incorrect type of video embeddings. "

983 + f"Got type: {type(video_embeds)}"

984 + )

985 + return OpenPanguVLVideoEmbeddingInputs(

986 + type="video_embeds",

987 + video_embeds=video_embeds,

988 + video_grid_thw=video_grid_thw,

989 + )

990 +

991 + def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:

992 + mm_input_by_modality = {}

993 + for input_key in kwargs:

994 + if (

995 + input_key in ("pixel_values", "image_embeds")

996 + and "image" not in mm_input_by_modality

997 + ):

998 + mm_input_by_modality["image"] = self._parse_and_validate_image_input(

999 + **kwargs

1000 + )

1001 + if (

1002 + input_key in ("pixel_values_videos", "video_embeds")

1003 + and "video" not in mm_input_by_modality

1004 + ):

1005 + mm_input_by_modality["video"] = self._parse_and_validate_video_input(

1006 + **kwargs

1007 + )

1008 + return mm_input_by_modality

1009 +

1010 + def get_language_model(self) -> torch.nn.Module:

1011 + return self.language_model

1012 +

1013 + def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings | None:

1014 + mm_input_by_modality = self._parse_and_validate_multimodal_inputs(**kwargs)

1015 + if not mm_input_by_modality:

1016 + return None

1017 +

1018 + multimodal_embeddings: tuple[torch.Tensor, ...] = ()

1019 +

1020 + for modality in mm_input_by_modality:

1021 + multimodal_input = mm_input_by_modality[modality]

1022 + if modality == "image":

1023 + vision_embeddings = self._process_image_input(multimodal_input)

1024 + multimodal_embeddings = (

1025 + multimodal_embeddings

1026 + if not vision_embeddings

1027 + else (multimodal_embeddings + vision_embeddings)

1028 + )

1029 + if modality == "video":

1030 + video_embeddings = self._process_video_input(multimodal_input)

1031 + multimodal_embeddings = (

1032 + multimodal_embeddings

1033 + if not video_embeddings

1034 + else (multimodal_embeddings + video_embeddings)

1035 + )

1036 + return multimodal_embeddings

1037 +

1038 + def get_input_embeddings(

1039 + self,

1040 + input_ids: torch.Tensor,

1041 + multimodal_embeddings=None,

1042 + ) -> torch.Tensor:

1043 + inputs_embeds = self.language_model.embed_input_ids(input_ids)

1044 + if multimodal_embeddings is not None:

1045 + inputs_embeds = self.embed_input_ids(

1046 + input_ids,

1047 + inputs_embeds,

1048 + multimodal_embeddings,

1049 + [self.config.image_token_id, self.config.video_token_id],

1050 + )

1051 + return inputs_embeds

1052 +

1053 + def _process_image_input(self, image_input) -> tuple[torch.Tensor, ...]:

1054 + grid_thw = image_input["image_grid_thw"]

1055 + if grid_thw.ndim != 2:

1056 + raise ValueError(f"grid_thw.ndim must be 2, but it is {grid_thw.ndim}")

1057 +

1058 + if image_input["type"] == "image_embeds":

1059 + image_embeds = image_input["image_embeds"].type(self.visual.dtype)

1060 + else:

1061 + pixel_values = image_input["pixel_values"].type(self.visual.dtype)

1062 + # rescale and normalize

1063 + pixel_values = pixel_values.reshape(

1064 + -1, self.channel, self.patch_size, self.patch_size

1065 + )

1066 + pixel_values = rescale_and_normalize(

1067 + pixel_values,

1068 + self.do_rescale,

1069 + self.rescale_factor,

1070 + self.do_normalize,

1071 + self.image_mean,

1072 + self.image_std,

1073 + )

1074 + pixel_values = pixel_values.reshape(

1075 + -1, self.channel * self.patch_size * self.patch_size

1076 + )

1077 + image_embeds = self.visual(pixel_values, grid_thw=grid_thw)

1078 +

1079 + # Split concatenated embeddings for each image item.

1080 + merge_size = self.visual.spatial_merge_size

1081 + sizes = grid_thw.prod(-1) // merge_size // merge_size

1082 + return image_embeds.split(sizes.tolist())

1083 +

1084 + def _process_video_input(self, video_input) -> torch.Tensor:

1085 + grid_thw = video_input["video_grid_thw"]

1086 + if grid_thw.ndim != 2:

1087 + raise ValueError(f"grid_thw.ndim must be 2, but it is {grid_thw.ndim}")

1088 +

1089 + if video_input["type"] == "video_embeds":

1090 + video_embeds = video_input["video_embeds"].type(self.visual.dtype)

1091 + else:

1092 + pixel_values_videos = video_input["pixel_values_videos"].type(

1093 + self.visual.dtype

1094 + )

1095 + video_embeds = self.visual(pixel_values_videos, grid_thw=grid_thw)

1096 +

1097 + # Split concatenated embeddings for each video item.

1098 + merge_size = self.visual.spatial_merge_size

1099 + sizes = grid_thw.prod(-1) // merge_size // merge_size

1100 +

1101 + return video_embeds.split(sizes.tolist())

1102 +

1103 + def forward(

1104 + self,

1105 + input_ids: torch.Tensor,

1106 + positions: torch.Tensor,

1107 + intermediate_tensors: IntermediateTensors | None = None,

1108 + inputs_embeds: torch.Tensor | None = None,

1109 + **kwargs: object,

1110 + ) -> torch.Tensor | IntermediateTensors:

1111 + if intermediate_tensors is not None:

1112 + inputs_embeds = None

1113 +

1114 + hidden_states = self.language_model.model(

1115 + input_ids=input_ids,

1116 + positions=positions,

1117 + intermediate_tensors=intermediate_tensors,

1118 + inputs_embeds=inputs_embeds,

1119 + )

1120 + return hidden_states

1121 +

1122 + def compute_logits(

1123 + self,

1124 + hidden_states: torch.Tensor,

1125 + sampling_metadata=None,

1126 + ) -> torch.Tensor | None:

1127 + return self.language_model.compute_logits(hidden_states)

1128 +

1129 + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:

1130 + loader = AutoWeightsLoader(self)

1131 + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)

1132 +

1133 + def get_mm_mapping(self) -> MultiModelKeys:

1134 + """

1135 + Get the module prefix in multimodal models

1136 + """

1137 + return MultiModelKeys.from_string_field(

1138 + language_model="language_model",

1139 + connector="visual.merger.",

1140 + tower_model="visual.",

1141 + )

1142 +

1143 + @classmethod

1144 + def get_placeholder_str(cls, modality: str, i: int) -> str | None:

1145 + if modality.startswith("image"):

1146 + return "[unused18][unused19][unused20]"

1147 + if modality.startswith("video"):

1148 + return "[unused18][unused32][unused20]"

1149 +

1150 + raise ValueError("Only image or video modality is supported")

1151 +

1152 + def iter_mm_grid_thw(

1153 + self, mm_features: list[MultiModalFeatureSpec]

1154 + ) -> Iterator[tuple[str, int, int, int, int]]:

1155 + spatial_merge_size = self.config.vision_config.spatial_merge_size

1156 + for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset):

1157 + offset = mm_feature.mm_position.offset

1158 + modality = mm_feature.modality

1159 + if modality == "image":

1160 + t, h, w = mm_feature.data["image_grid_thw"].data.tolist()

1161 + assert t == 1, f"Image must have 1 frame, got {t}"

1162 + yield (

1163 + modality,

1164 + offset,

1165 + 1,

1166 + h // spatial_merge_size,

1167 + w // spatial_merge_size,

1168 + )

1169 + elif modality == "video":

1170 + t, h, w = mm_feature.data["video_grid_thw"].data.tolist()

1171 + yield (

1172 + modality,

1173 + offset,

1174 + t,

1175 + h // spatial_merge_size,

1176 + w // spatial_merge_size,

1177 + )

1178 + else:

1179 + raise ValueError(f"Unsupported modality: {modality}")

1180 +

1181 + def get_mrope_input_positions(

1182 + self,

1183 + input_tokens: list[int],

1184 + mm_features: list[MultiModalFeatureSpec],

1185 + ) -> tuple[torch.Tensor, int]:

1186 + llm_pos_ids_list: list = []

1187 + st = 0

1188 +

1189 + for (

1190 + modality,

1191 + offset,

1192 + llm_grid_t,

1193 + llm_grid_h,

1194 + llm_grid_w,

1195 + ) in self.iter_mm_grid_thw(mm_features):

1196 + text_len = offset - st

1197 + st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0

1198 + llm_pos_ids_list.append(

1199 + torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx

1200 + )

1201 + if modality == "video":

1202 + eot_bot_pos = torch.full((3, 1), 0, dtype=torch.long)

1203 + offset_pos = max(llm_grid_h, llm_grid_w)

1204 + current_pos = text_len + st_idx

1205 + grid_h = (

1206 + torch.arange(llm_grid_h)

1207 + .view(-1, 1)

1208 + .expand(-1, llm_grid_w)

1209 + .flatten()

1210 + )

1211 + grid_w = (

1212 + torch.arange(llm_grid_w)

1213 + .view(1, -1)

1214 + .expand(llm_grid_h, -1)

1215 + .flatten()

1216 + )

1217 + frame_pos = torch.stack(

1218 + [

1219 + torch.full_like(grid_h, 0, dtype=torch.long), # t

1220 + grid_h, # h

1221 + grid_w, # w

1222 + ]

1223 + )

1224 + llm_pos_ids_list.append(frame_pos + current_pos)

1225 + for _ in range(llm_grid_t - 1):

1226 + current_pos = current_pos + offset_pos

1227 + llm_pos_ids_list.append(eot_bot_pos + current_pos)

1228 + llm_pos_ids_list.append(eot_bot_pos + current_pos + 1)

1229 + llm_pos_ids_list.append(frame_pos + current_pos + 2)

1230 + current_pos += 2

1231 + st = (

1232 + offset + llm_grid_t * llm_grid_h * llm_grid_w + (llm_grid_t - 1) * 2

1233 + )

1234 + else:

1235 + t_index = (

1236 + (

1237 + torch.arange(llm_grid_t)

1238 + .view(-1, 1)

1239 + .expand(-1, llm_grid_h * llm_grid_w)

1240 + )

1241 + .long()

1242 + .flatten()

1243 + )

1244 + h_index = (

1245 + torch.arange(llm_grid_h)

1246 + .view(1, -1, 1)

1247 + .expand(llm_grid_t, -1, llm_grid_w)

1248 + .flatten()

1249 + )

1250 + w_index = (

1251 + torch.arange(llm_grid_w)

1252 + .view(1, 1, -1)

1253 + .expand(llm_grid_t, llm_grid_h, -1)

1254 + .flatten()

1255 + )

1256 + llm_pos_ids_list.append(

1257 + torch.stack([t_index, h_index, w_index]) + text_len + st_idx

1258 + )

1259 + st = offset + llm_grid_t * llm_grid_h * llm_grid_w

1260 + if st < len(input_tokens):

1261 + st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0

1262 + text_len = len(input_tokens) - st

1263 + llm_pos_ids_list.append(

1264 + torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx

1265 + )

1266 + llm_positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1)

1267 + mrope_position_delta = (llm_positions.max() + 1 - len(input_tokens)).item()

1268 + return llm_positions, mrope_position_delta

1269 +

1270 +

1271 +def rescale(image, scale):

1272 + return image * scale

1273 +

1274 +

1275 +def normalize(image, mean, std):

1276 + return v2.functional.normalize(image, mean, std)

1277 +

1278 +

1279 +@lru_cache(maxsize=10)

1280 +def _fuse_mean_std_and_rescale_factor(

1281 + do_normalize: bool | None = None,

1282 + image_mean: float | list[float] | None = None,

1283 + image_std: float | list[float] | None = None,

1284 + do_rescale: bool | None = None,

1285 + rescale_factor: float | None = None,

1286 + device: Optional["torch.device"] = None,

1287 +) -> tuple:

1288 + if do_rescale and do_normalize:

1289 + # Fused rescale and normalize

1290 + image_mean = torch.tensor(image_mean, device=device) * (1.0 / rescale_factor)

1291 + image_std = torch.tensor(image_std, device=device) * (1.0 / rescale_factor)

1292 + do_rescale = False

1293 + return image_mean, image_std, do_rescale

1294 +

1295 +

1296 +def rescale_and_normalize(

1297 + images: "torch.Tensor",

1298 + do_rescale: bool,

1299 + rescale_factor: float,

1300 + do_normalize: bool,

1301 + image_mean: float | list[float],

1302 + image_std: float | list[float],

1303 + dtype: torch.dtype = torch.bfloat16,

1304 +) -> "torch.Tensor":

1305 + """

1306 + Rescale and normalize images.

1307 + """

1308 + image_mean, image_std, do_rescale = _fuse_mean_std_and_rescale_factor(

1309 + do_normalize=do_normalize,

1310 + image_mean=image_mean,

1311 + image_std=image_std,

1312 + do_rescale=do_rescale,

1313 + rescale_factor=rescale_factor,

1314 + device=images.device,

1315 + )

1316 + # if/elif as we use fused rescale and normalize if both are set to True

1317 + if do_normalize:

1318 + images = normalize(images.to(dtype=torch.float32), image_mean, image_std)

1319 + elif do_rescale:

1320 + images = rescale(images, rescale_factor)

1321 + images = images.to(dtype)

1322 +

1323 + return images

vllm/model_executor/models/registry.py

@@ -405,6 +405,10 @@ _MULTIMODAL_MODELS = {

405 "MolmoForCausalLM": ("molmo", "MolmoForCausalLM"),

406 "Molmo2ForConditionalGeneration": ("molmo2", "Molmo2ForConditionalGeneration"),

407 "NVLM_D": ("nvlm_d", "NVLM_D_Model"),

408 + "OpenPanguVLForConditionalGeneration": (

409 + "openpangu_vl",

410 + "OpenPanguVLForConditionalGeneration",

411 + ),

412 "Ovis": ("ovis", "Ovis"),

413 "Ovis2_5": ("ovis2_5", "Ovis2_5"),

414 "PaddleOCRVLForConditionalGeneration": (