@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0f7acdd73c

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

[Model] Support Qwen3-VL Model Series (#24727)

Signed-off-by: Roger Wang <hey@rogerw.io> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Huang Jie <92386084+JJJYmmm@users.noreply.github.com> Co-authored-by: 松灵 <26085463+wulipc@users.noreply.github.com> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>

Roger Wang committed Sep 16, 2025 at 22:01 UTC 0f7acdd73ca6316c8ae0474c0a9c4fc264e87a7b

13 files changed +2084 -17

docs/models/supported_models.md

@@ -661,6 +661,8 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen

661 | `Qwen2VLForConditionalGeneration` | QVQ, Qwen2-VL | T + IE+ + VE+ | `Qwen/QVQ-72B-Preview`, `Qwen/Qwen2-VL-7B-Instruct`, `Qwen/Qwen2-VL-72B-Instruct`, etc. | ✅︎ | ✅︎ | ✅︎ |

662 | `Qwen2_5_VLForConditionalGeneration` | Qwen2.5-VL | T + IE+ + VE+ | `Qwen/Qwen2.5-VL-3B-Instruct`, `Qwen/Qwen2.5-VL-72B-Instruct`, etc. | ✅︎ | ✅︎ | ✅︎ |

663 | `Qwen2_5OmniThinkerForConditionalGeneration` | Qwen2.5-Omni | T + IE+ + VE+ + A+ | `Qwen/Qwen2.5-Omni-3B`, `Qwen/Qwen2.5-Omni-7B` | ✅︎ | ✅︎ | ✅︎ |

664 +| `Qwen3VLForConditionalGeneration` | Qwen3-VL | T + IE+ + VE+ | `Qwen/Qwen3-VL-4B-Instruct`, etc. | ✅︎ | ✅︎ | ✅︎ |

665 +| `Qwen3VLMoeForConditionalGeneration` | Qwen3-VL-MOE | T + IE+ + VE+ | `Qwen/Qwen3-VL-30B-A3B-Instruct`, etc. | ✅︎ | ✅︎ | ✅︎ |

666 | `RForConditionalGeneration` | R-VL-4B | T + IE+ | `YannQi/R-4B` | | ✅︎ | ✅︎ |

667 | `SkyworkR1VChatModel` | Skywork-R1V-38B | T + I | `Skywork/Skywork-R1V-38B` | | ✅︎ | ✅︎ |

668 | `SmolVLMForConditionalGeneration` | SmolVLM2 | T + I | `SmolVLM2-2.2B-Instruct` | ✅︎ | | ✅︎ |

examples/offline_inference/vision_language.py

+78

@@ -1437,6 +1437,80 @@ def run_qwen2_5_omni(questions: list[str], modality: str):

1437 )

1438

1439

1440 +# Qwen3-VL-Dense

1441 +def run_qwen3_vl(questions: list[str], modality: str) -> ModelRequestData:

1442 + model_name = "Qwen/Qwen3-VL-4B-Instruct"

1443 +

1444 + engine_args = EngineArgs(

1445 + model=model_name,

1446 + max_model_len=4096,

1447 + max_num_seqs=5,

1448 + mm_processor_kwargs={

1449 + "min_pixels": 28 * 28,

1450 + "max_pixels": 1280 * 28 * 28,

1451 + "fps": 1,

1452 + },

1453 + limit_mm_per_prompt={modality: 1},

1454 + )

1455 +

1456 + if modality == "image":

1457 + placeholder = "<|image_pad|>"

1458 + elif modality == "video":

1459 + placeholder = "<|video_pad|>"

1460 +

1461 + prompts = [

1462 + (

1463 + "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"

1464 + f"<|im_start|>user\n<|vision_start|>{placeholder}<|vision_end|>"

1465 + f"{question}<|im_end|>\n"

1466 + "<|im_start|>assistant\n"

1467 + )

1468 + for question in questions

1469 + ]

1470 +

1471 + return ModelRequestData(

1472 + engine_args=engine_args,

1473 + prompts=prompts,

1474 + )

1475 +

1476 +

1477 +# Qwen3-VL-MOE

1478 +def run_qwen3_vl_moe(questions: list[str], modality: str) -> ModelRequestData:

1479 + model_name = "Qwen/Qwen3-VL-30B-A3B-Instruct"

1480 +

1481 + engine_args = EngineArgs(

1482 + model=model_name,

1483 + max_model_len=4096,

1484 + max_num_seqs=5,

1485 + mm_processor_kwargs={

1486 + "min_pixels": 28 * 28,

1487 + "max_pixels": 1280 * 28 * 28,

1488 + "fps": 1,

1489 + },

1490 + limit_mm_per_prompt={modality: 1},

1491 + )

1492 +

1493 + if modality == "image":

1494 + placeholder = "<|image_pad|>"

1495 + elif modality == "video":

1496 + placeholder = "<|video_pad|>"

1497 +

1498 + prompts = [

1499 + (

1500 + "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"

1501 + f"<|im_start|>user\n<|vision_start|>{placeholder}<|vision_end|>"

1502 + f"{question}<|im_end|>\n"

1503 + "<|im_start|>assistant\n"

1504 + )

1505 + for question in questions

1506 + ]

1507 +

1508 + return ModelRequestData(

1509 + engine_args=engine_args,

1510 + prompts=prompts,

1511 + )

1512 +

1513 +

1514 # R-4B

1515 def run_r_vl(questions: list[str], modality: str) -> ModelRequestData:

1516 assert modality == "image"

@@ -1645,6 +1719,8 @@ model_example_map = {

1719 "qwen2_vl": run_qwen2_vl,

1720 "qwen2_5_vl": run_qwen2_5_vl,

1721 "qwen2_5_omni": run_qwen2_5_omni,

1722 + "qwen3_vl": run_qwen3_vl,

1723 + "qwen3_vl_moe": run_qwen3_vl_moe,

1724 "rvl": run_r_vl,

1725 "skywork_chat": run_skyworkr1v,

1726 "smolvlm": run_smolvlm,

@@ -1658,6 +1734,8 @@ MODELS_NEED_VIDEO_METADATA = [

1734 "glm4_1v",

1735 "glm4_5v",

1736 "glm4_5v_fp8",

1737 + "qwen3_vl",

1738 + "qwen3_vl_moe",

1739 ]

1740

1741

tests/models/multimodal/processing/test_common.py

+34 -1

@@ -31,6 +31,7 @@ def glm4_1v_patch_mm_data(mm_data: MultiModalDataDict) -> MultiModalDataDict:

31 """

32 # Ensure video metadata is included

33 if "video" in mm_data:

34 + # GLM4.1V doesn't support multiple videos

35 video = mm_data["video"]

36 num_frames = len(video)

37 mm_data["video"] = (video, {

@@ -44,6 +45,34 @@ def glm4_1v_patch_mm_data(mm_data: MultiModalDataDict) -> MultiModalDataDict:

45 return mm_data

48 +def qwen3_vl_patch_mm_data(mm_data: MultiModalDataDict) -> MultiModalDataDict:

49 + """

50 + Patch the multimodal data for Qwen3-VL model.

51 + """

52 +

53 + def create_metadata(frames: np.ndarray):

54 + num_frames = len(frames)

55 + return {

56 + "total_num_frames": num_frames,

57 + "fps": 2.0,

58 + "duration": num_frames / 2.0,

59 + "video_backend": "opencv",

60 + "frames_indices": list(range(num_frames)),

61 + "do_sample_frames": True,

62 + }

63 +

64 + # Ensure video metadata is included

65 + if "video" in mm_data:

66 + video = mm_data["video"]

67 + if isinstance(video, list):

68 + # multiple videos

69 + mm_data["video"] = [(vid, create_metadata(vid)) for vid in video]

70 + else:

71 + # single video

72 + mm_data["video"] = (video, create_metadata(video))

73 + return mm_data

74 +

75 +

76 def _test_processing_correctness(

77 model_id_or_arch: str,

78 hit_rate: float,

@@ -182,8 +211,10 @@ _IGNORE_MM_KEYS = {

211 }

212

213 MM_DATA_PATCHES = {

185 - # GLM4.1V requires video metadata to be included in the input

214 + # GLM4.1V and Qwen3-VL requires video metadata to be included in the input

215 "glm4v": glm4_1v_patch_mm_data,

216 + "qwen3_vl": qwen3_vl_patch_mm_data,

217 + "qwen3_vl_moe": qwen3_vl_patch_mm_data,

218 }

219

220

@@ -326,6 +357,8 @@ def _test_processing_correctness_one(

357 "Qwen/Qwen2.5-VL-3B-Instruct",

358 "Qwen/Qwen2-Audio-7B-Instruct",

359 "Qwen/Qwen2.5-Omni-3B",

360 + "Qwen/Qwen3-VL-4B-Instruct",

361 + "Qwen/Qwen3-VL-30B-A3B-Instruct",

362 "YannQi/R-4B",

363 "Skywork/Skywork-R1V-38B",

364 "HuggingFaceTB/SmolVLM2-2.2B-Instruct",

tests/models/registry.py

@@ -557,6 +557,12 @@ _MULTIMODAL_EXAMPLE_MODELS = {

557 max_model_len=4096),

558 "Qwen2_5OmniModel": _HfExamplesInfo("Qwen/Qwen2.5-Omni-3B"),

559 "Qwen2_5OmniForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2.5-Omni-7B-AWQ"), # noqa: E501

560 + "Qwen3VLForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen3-VL-4B-Instruct", # noqa: E501

561 + max_model_len=4096,

562 + min_transformers_version="4.57"), # noqa: E501

563 + "Qwen3VLMoeForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen3-VL-30B-A3B-Instruct", # noqa: E501

564 + max_model_len=4096,

565 + min_transformers_version="4.57"),

566 "RForConditionalGeneration": _HfExamplesInfo("YannQi/R-4B",

567 trust_remote_code=True),

568 "SkyworkR1VChatModel": _HfExamplesInfo("Skywork/Skywork-R1V-38B",

vllm/model_executor/layers/rotary_embedding/__init__.py

@@ -103,6 +103,8 @@ def get_rope(

103 is_neox_style,

104 dtype,

105 mrope_section=rope_scaling["mrope_section"],

106 + mrope_interleaved=rope_scaling.get("mrope_interleaved",

107 + False),

108 )

109 else:

110 rotary_emb = RotaryEmbedding(

vllm/model_executor/layers/rotary_embedding/mrope.py

+133 -11

@@ -177,6 +177,18 @@ def triton_mrope(

177 return q, k

178

179

180 +def apply_interleaved_rope(x: torch.Tensor,

181 + mrope_section: list[int]) -> torch.Tensor:

182 + """Apply interleaved MRoPE to 3D rotary embeddings.

183 + Reorganizes frequency layout from chunked [TTT...HHH...WWW] to

184 + interleaved [THTHWHTHW...TT], preserving frequency continuity.

185 + """

186 + x_t = x[0].clone()

187 + x_t[..., 1:mrope_section[1] * 3:3] = x[1, ..., 1:mrope_section[1] * 3:3]

188 + x_t[..., 2:mrope_section[2] * 3:3] = x[2, ..., 2:mrope_section[2] * 3:3]

189 + return x_t

190 +

191 +

192 class MRotaryEmbedding(RotaryEmbedding):

193 """Rotary Embedding with Multimodal Sections."""

194

@@ -189,6 +201,7 @@ class MRotaryEmbedding(RotaryEmbedding):

201 is_neox_style: bool,

202 dtype: torch.dtype,

203 mrope_section: Optional[list[int]] = None,

204 + mrope_interleaved: Optional[bool] = False,

205 ) -> None:

206 # In Qwen2.5-VL, the maximum index value is related to the duration of

207 # the input video. We enlarge max_position_embeddings to 4 times to get

@@ -198,6 +211,7 @@ class MRotaryEmbedding(RotaryEmbedding):

211 base, is_neox_style, dtype)

212

213 self.mrope_section = mrope_section

214 + self.mrope_interleaved = mrope_interleaved

215 if self.mrope_section:

216 assert sum(self.mrope_section) == rotary_dim // 2

217

@@ -225,17 +239,20 @@ class MRotaryEmbedding(RotaryEmbedding):

239 cos, sin = cos_sin.chunk(2, dim=-1)

240 if positions.ndim == 2:

241 assert self.mrope_section

228 -

229 - cos = torch.cat([

230 - m[i]

231 - for i, m in enumerate(cos.split(self.mrope_section, dim=-1))

232 - ],

233 - dim=-1)

234 - sin = torch.cat([

235 - m[i]

236 - for i, m in enumerate(sin.split(self.mrope_section, dim=-1))

237 - ],

238 - dim=-1)

242 + if self.mrope_interleaved:

243 + cos = apply_interleaved_rope(cos, self.mrope_section)

244 + sin = apply_interleaved_rope(sin, self.mrope_section)

245 + else:

246 + cos = torch.cat([

247 + m[i] for i, m in enumerate(

248 + cos.split(self.mrope_section, dim=-1))

249 + ],

250 + dim=-1)

251 + sin = torch.cat([

252 + m[i] for i, m in enumerate(

253 + sin.split(self.mrope_section, dim=-1))

254 + ],

255 + dim=-1)

256

257 query_shape = query.shape

258 query = query.view(num_tokens, -1, self.head_size)

@@ -265,6 +282,10 @@ class MRotaryEmbedding(RotaryEmbedding):

282 assert positions.ndim == 1 or positions.ndim == 2

283 assert key is not None

284

285 + if self.mrope_interleaved:

286 + # TODO: add triton implementation to support mrope-interleaved

287 + return self.forward_native(positions, query, key)

288 +

289 num_tokens = positions.shape[-1]

290 cos_sin = self.cos_sin_cache[positions]

291 cos, sin = cos_sin.chunk(2, dim=-1)

@@ -388,6 +409,15 @@ class MRotaryEmbedding(RotaryEmbedding):

409 context_len=context_len,

410 seq_len=seq_len,

411 )

412 + elif hf_config.model_type in ["qwen3_vl", "qwen3_vl_moe"]:

413 + return cls._qwen3vl_get_input_positions_tensor(

414 + input_tokens=input_tokens,

415 + hf_config=hf_config,

416 + image_grid_thw=image_grid_thw,

417 + video_grid_thw=video_grid_thw,

418 + context_len=context_len,

419 + seq_len=seq_len,

420 + )

421 elif hf_config.model_type in ["ernie4_5_moe_vl", "ernie4_5_vl"]:

422 return cls._ernie_get_input_positions_tensor(

423 input_tokens=input_tokens,

@@ -526,6 +556,98 @@ class MRotaryEmbedding(RotaryEmbedding):

556 len(input_tokens)).item()

557 return llm_positions, mrope_position_delta

558

559 + @classmethod

560 + def _qwen3vl_get_input_positions_tensor(

561 + cls,

562 + input_tokens: list[int],

563 + hf_config: PretrainedConfig,

564 + image_grid_thw: Union[list[list[int]], torch.Tensor],

565 + video_grid_thw: Union[list[list[int]], torch.Tensor],

566 + context_len: int = 0,

567 + seq_len: Optional[int] = None,

568 + ) -> tuple[torch.Tensor, int]:

569 + """Get mrope input positions and delta value."""

570 +

571 + video_grid_thw = [[1, h, w] for t, h, w in video_grid_thw

572 + for _ in range(t)]

573 +

574 + image_token_id = hf_config.image_token_id

575 + video_token_id = hf_config.video_token_id

576 + vision_start_token_id = hf_config.vision_start_token_id

577 + spatial_merge_size = hf_config.vision_config.spatial_merge_size

578 +

579 + input_tokens_tensor = torch.tensor(input_tokens)

580 + vision_start_indices = torch.argwhere(

581 + input_tokens_tensor == vision_start_token_id).squeeze(1)

582 + vision_tokens = input_tokens_tensor[vision_start_indices + 1]

583 + image_nums = (vision_tokens == image_token_id).sum()

584 + video_nums = (vision_tokens == video_token_id).sum()

585 + llm_pos_ids_list: list = []

586 +

587 + st = 0

588 + remain_images, remain_videos = image_nums, video_nums

589 +

590 + image_index, video_index = 0, 0

591 + for _ in range(image_nums + video_nums):

592 + if image_token_id in input_tokens and remain_images > 0:

593 + ed_image = input_tokens.index(image_token_id, st)

594 + else:

595 + ed_image = len(input_tokens) + 1

596 + if video_token_id in input_tokens and remain_videos > 0:

597 + ed_video = input_tokens.index(video_token_id, st)

598 + else:

599 + ed_video = len(input_tokens) + 1

600 + if ed_image < ed_video:

601 + t, h, w = (

602 + image_grid_thw[image_index][0],

603 + image_grid_thw[image_index][1],

604 + image_grid_thw[image_index][2],

605 + )

606 + image_index += 1

607 + remain_images -= 1

608 + ed = ed_image

609 + else:

610 + t, h, w = (

611 + video_grid_thw[video_index][0],

612 + video_grid_thw[video_index][1],

613 + video_grid_thw[video_index][2],

614 + )

615 + video_index += 1

616 + remain_videos -= 1

617 + ed = ed_video

618 +

619 + llm_grid_t, llm_grid_h, llm_grid_w = \

620 + t, h // spatial_merge_size, w // spatial_merge_size

621 + text_len = ed - st

622 +

623 + st_idx = llm_pos_ids_list[-1].max() + 1 if len(

624 + llm_pos_ids_list) > 0 else 0

625 + llm_pos_ids_list.append(

626 + torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx)

627 +

628 + t_index = torch.arange(llm_grid_t).view(-1, 1).expand(

629 + -1, llm_grid_h * llm_grid_w).flatten()

630 + h_index = torch.arange(llm_grid_h).view(1, -1, 1).expand(

631 + llm_grid_t, -1, llm_grid_w).flatten()

632 + w_index = torch.arange(llm_grid_w).view(1, 1, -1).expand(

633 + llm_grid_t, llm_grid_h, -1).flatten()

634 + llm_pos_ids_list.append(

635 + torch.stack([t_index, h_index, w_index]) + text_len + st_idx)

636 + st = ed + llm_grid_t * llm_grid_h * llm_grid_w

637 +

638 + if st < len(input_tokens):

639 + st_idx = llm_pos_ids_list[-1].max() + 1 if len(

640 + llm_pos_ids_list) > 0 else 0

641 + text_len = len(input_tokens) - st

642 + llm_pos_ids_list.append(

643 + torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx)

644 +

645 + llm_positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1)

646 + mrope_position_delta = (llm_positions.max() + 1 -

647 + len(input_tokens)).item()

648 + llm_positions = llm_positions[:, context_len:seq_len]

649 + return llm_positions, mrope_position_delta

650 +

651 @classmethod

652 def _ernie_get_input_positions_tensor(

653 cls,

vllm/model_executor/models/qwen2.py

+1 -1

@@ -285,7 +285,7 @@ class Qwen2Model(nn.Module):

285 decoder_layer_type: type[nn.Module] = Qwen2DecoderLayer):

286 super().__init__()

287

288 - config = vllm_config.model_config.hf_config

288 + config = vllm_config.model_config.hf_config.get_text_config()

289 cache_config = vllm_config.cache_config

290 quant_config = vllm_config.quant_config

291

vllm/model_executor/models/qwen2_vl.py

+1 -1

@@ -83,7 +83,7 @@ from .vision import get_vit_attn_backend

83 logger = init_logger(__name__)

85 # For profile run

86 -_MAX_FRAMES_PER_VIDEO = 16

86 +_MAX_FRAMES_PER_VIDEO = 600

88 # === Vision Inputs === #

vllm/model_executor/models/qwen3_moe.py

+1 -1

@@ -378,7 +378,7 @@ class Qwen3MoeModel(nn.Module):

378 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):

379 super().__init__()

380

381 - config = vllm_config.model_config.hf_config

381 + config = vllm_config.model_config.hf_config.get_text_config()

382 cache_config = vllm_config.cache_config

383 quant_config = vllm_config.quant_config

384 parallel_config = vllm_config.parallel_config

vllm/model_executor/models/qwen3_vl.py new

+1478

@@ -0,0 +1,1478 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +

8 +#

9 +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX

10 +# and OPT implementations in this library. It has been modified from its

11 +# original forms to accommodate minor architectural differences compared

12 +# to GPT-NeoX and OPT used by the Meta AI team that trained the model.

13 +#

14 +# Licensed under the Apache License, Version 2.0 (the "License");

15 +# you may not use this file except in compliance with the License.

16 +# You may obtain a copy of the License at

17 +#

18 +# http://www.apache.org/licenses/LICENSE-2.0

19 +#

20 +# Unless required by applicable law or agreed to in writing, software

21 +# distributed under the License is distributed on an "AS IS" BASIS,

22 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

23 +# See the License for the specific language governing permissions and

24 +# limitations under the License.

25 +"""Inference-only Qwen3VL model compatible with HuggingFace weights."""

26 +from collections.abc import Iterable, Mapping, Sequence

27 +from functools import partial

28 +from typing import Any, Callable, Optional, Union

29 +

30 +import numpy as np

31 +import torch

32 +import torch.nn as nn

33 +import torch.nn.functional as F

34 +from transformers import BatchFeature

35 +from transformers.models.qwen2_vl import Qwen2VLImageProcessorFast

36 +from transformers.models.qwen2_vl.image_processing_qwen2_vl import smart_resize

37 +from transformers.models.qwen3_vl import (Qwen3VLProcessor,

38 + Qwen3VLVideoProcessor)

39 +from transformers.models.qwen3_vl.configuration_qwen3_vl import (

40 + Qwen3VLConfig, Qwen3VLVisionConfig)

41 +from transformers.video_utils import VideoMetadata

42 +

43 +from vllm.attention.layer import check_upstream_fa_availability

44 +from vllm.compilation.decorators import support_torch_compile

45 +from vllm.config import VllmConfig

46 +from vllm.distributed import get_pp_group

47 +from vllm.logger import init_logger

48 +from vllm.model_executor import SamplingMetadata

49 +from vllm.model_executor.layers.activation import _ACTIVATION_REGISTRY

50 +from vllm.model_executor.layers.linear import (ColumnParallelLinear,

51 + RowParallelLinear)

52 +from vllm.model_executor.layers.logits_processor import LogitsProcessor

53 +from vllm.model_executor.layers.quantization import QuantizationConfig

54 +from vllm.model_executor.layers.quantization.gptq import GPTQConfig

55 +from vllm.model_executor.layers.quantization.gptq_marlin import (

56 + GPTQMarlinConfig)

57 +from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead

58 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader

59 +from vllm.model_executor.models.module_mapping import MultiModelKeys

60 +from vllm.multimodal import MULTIMODAL_REGISTRY

61 +from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,

62 + MultiModalKwargsItem,

63 + MultiModalKwargsItems, VideoItem)

64 +from vllm.multimodal.parse import (ImageSize, MultiModalDataItems,

65 + MultiModalDataParser)

66 +from vllm.multimodal.processing import (BaseMultiModalProcessor,

67 + PromptReplacement, PromptUpdate,

68 + PromptUpdateDetails)

69 +from vllm.multimodal.profiling import BaseDummyInputsBuilder

70 +from vllm.platforms import _Backend

71 +from vllm.sequence import IntermediateTensors

72 +from vllm.transformers_utils.config import uses_mrope

73 +from vllm.utils import is_list_of

74 +

75 +from .interfaces import (MultiModalEmbeddings, SupportsLoRA,

76 + SupportsMultiModal, SupportsPP)

77 +from .qwen2_5_vl import (Qwen2_5_VisionAttention,

78 + Qwen2_5_VisionRotaryEmbedding,

79 + Qwen2_5_VLImageEmbeddingInputs, Qwen2_5_VLImageInputs,

80 + Qwen2_5_VLImagePixelInputs,

81 + Qwen2_5_VLVideoEmbeddingInputs, Qwen2_5_VLVideoInputs,

82 + Qwen2_5_VLVideoPixelInputs)

83 +from .qwen2_vl import Qwen2VLProcessingInfo

84 +from .qwen3 import Qwen3ForCausalLM, Qwen3Model

85 +from .utils import (AutoWeightsLoader, PPMissingLayer, WeightsMapper,

86 + maybe_prefix, merge_multimodal_embeddings)

87 +from .vision import get_vit_attn_backend

88 +

89 +logger = init_logger(__name__)

90 +

91 +

92 +class Qwen3_VisionPatchEmbed(nn.Module):

93 +

94 + def __init__(

95 + self,

96 + patch_size: int = 14,

97 + temporal_patch_size: int = 2,

98 + in_channels: int = 3,

99 + hidden_size: int = 1152,

100 + ) -> None:

101 + super().__init__()

102 + self.patch_size = patch_size

103 + self.temporal_patch_size = temporal_patch_size

104 + self.hidden_size = hidden_size

105 +

106 + kernel_size = (temporal_patch_size, patch_size, patch_size)

107 + self.proj = nn.Conv3d(in_channels,

108 + hidden_size,

109 + kernel_size=kernel_size,

110 + stride=kernel_size,

111 + bias=True)

112 +

113 + def forward(self, x: torch.Tensor) -> torch.Tensor:

114 + L, C = x.shape

115 + x = x.view(L, -1, self.temporal_patch_size, self.patch_size,

116 + self.patch_size)

117 + x = self.proj(x).view(L, self.hidden_size)

118 + return x

119 +

120 +

121 +class Qwen3_VisionMLP(nn.Module):

122 +

123 + def __init__(self,

124 + in_features: int,

125 + hidden_features: int,

126 + bias: bool = False,

127 + act_fn: Callable[[torch.Tensor], torch.Tensor] = F.silu,

128 + quant_config: Optional[QuantizationConfig] = None,

129 + prefix: str = ""):

130 + super().__init__()

131 + self.linear_fc1 = ColumnParallelLinear(in_features,

132 + hidden_features,

133 + bias=bias,

134 + quant_config=quant_config,

135 + return_bias=False,

136 + prefix=f"{prefix}.linear_fc1")

137 + self.linear_fc2 = RowParallelLinear(hidden_features,

138 + in_features,

139 + bias=bias,

140 + quant_config=quant_config,

141 + return_bias=False,

142 + prefix=f"{prefix}.linear_fc2")

143 + self.act_fn = act_fn

144 +

145 + def forward(self, x: torch.Tensor):

146 + mlp_output = self.linear_fc2(self.act_fn(self.linear_fc1(x)))

147 + return mlp_output

148 +

149 +

150 +class Qwen3_VisionBlock(nn.Module):

151 +

152 + def __init__(

153 + self,

154 + dim: int,

155 + num_heads: int,

156 + mlp_hidden_dim: int,

157 + act_fn: Callable[[torch.Tensor], torch.Tensor] = F.silu,

158 + norm_layer: Optional[Callable[[int], nn.Module]] = None,

159 + quant_config: Optional[QuantizationConfig] = None,

160 + prefix: str = "",

161 + ) -> None:

162 + super().__init__()

163 + if norm_layer is None:

164 + norm_layer = partial(nn.LayerNorm, eps=1e-6)

165 + self.norm1 = norm_layer(dim)

166 + self.norm2 = norm_layer(dim)

167 + self.attn = Qwen2_5_VisionAttention(embed_dim=dim,

168 + num_heads=num_heads,

169 + projection_size=dim,

170 + quant_config=quant_config,

171 + prefix=f"{prefix}.attn")

172 + self.mlp = Qwen3_VisionMLP(dim,

173 + mlp_hidden_dim,

174 + act_fn=act_fn,

175 + bias=True,

176 + quant_config=quant_config,

177 + prefix=f"{prefix}.mlp")

178 +

179 + def forward(

180 + self,

181 + x: torch.Tensor,

182 + cu_seqlens: torch.Tensor,

183 + rotary_pos_emb: torch.Tensor,

184 + max_seqlen: Optional[int] = None, # Only used for Flash Attention

185 + seqlens: Optional[list[int]] = None, # Only used for xFormers

186 + ) -> torch.Tensor:

187 + x = x + self.attn(self.norm1(x),

188 + cu_seqlens=cu_seqlens,

189 + rotary_pos_emb=rotary_pos_emb,

190 + max_seqlen=max_seqlen,

191 + seqlens=seqlens)

192 +

193 + x = x + self.mlp(self.norm2(x))

194 + return x

195 +

196 +

197 +class Qwen3_VisionPatchMerger(nn.Module):

198 +

199 + def __init__(

200 + self,

201 + d_model: int,

202 + context_dim: int,

203 + norm_layer: Optional[Callable[[int], nn.Module]] = None,

204 + spatial_merge_size: int = 2,

205 + use_postshuffle_norm: bool = False,

206 + quant_config: Optional[QuantizationConfig] = None,

207 + prefix: str = "",

208 + ) -> None:

209 + super().__init__()

210 + self.hidden_size = context_dim * (spatial_merge_size**2)

211 +

212 + self.use_postshuffle_norm = use_postshuffle_norm

213 + if self.use_postshuffle_norm:

214 + context_dim = self.hidden_size

215 +

216 + if norm_layer is None:

217 + norm_layer = partial(nn.LayerNorm, eps=1e-6)

218 + self.use_postshuffle_norm = use_postshuffle_norm

219 + self.norm = norm_layer(

220 + self.hidden_size if use_postshuffle_norm else context_dim)

221 + self.linear_fc1 = ColumnParallelLinear(self.hidden_size,

222 + self.hidden_size,

223 + bias=True,

224 + quant_config=quant_config,

225 + prefix=f"{prefix}.linear_fc1")

226 + self.act_fn = nn.GELU()

227 + self.linear_fc2 = RowParallelLinear(self.hidden_size,

228 + d_model,

229 + bias=True,

230 + quant_config=quant_config,

231 + prefix=f"{prefix}.linear_fc2")

232 +

233 + def forward(self, x: torch.Tensor) -> torch.Tensor:

234 + if self.use_postshuffle_norm:

235 + x = self.norm(x.view(-1, self.hidden_size))

236 + else:

237 + x = self.norm(x).view(-1, self.hidden_size)

238 +

239 + x_parallel, _ = self.linear_fc1(x)

240 + x_parallel = self.act_fn(x_parallel)

241 + out, _ = self.linear_fc2(x_parallel)

242 + return out

243 +

244 +

245 +class Qwen3_VisionTransformer(nn.Module):

246 +

247 + def __init__(

248 + self,

249 + vision_config: Qwen3VLVisionConfig,

250 + norm_eps: float = 1e-6,

251 + quant_config: Optional[QuantizationConfig] = None,

252 + prefix: str = "",

253 + ) -> None:

254 + super().__init__()

255 + self.hidden_size = vision_config.hidden_size

256 + self.num_heads = vision_config.num_heads

257 + self.num_position_embeddings = vision_config.num_position_embeddings

258 + self.patch_size = vision_config.patch_size

259 + self.spatial_merge_size = vision_config.spatial_merge_size

260 + self.spatial_merge_unit = self.spatial_merge_size**2

261 + self.temporal_patch_size = vision_config.temporal_patch_size

262 + self.deepstack_visual_indexes = vision_config.deepstack_visual_indexes

263 +

264 + self.patch_embed = Qwen3_VisionPatchEmbed(

265 + patch_size=self.patch_size,

266 + temporal_patch_size=self.temporal_patch_size,

267 + in_channels=vision_config.in_channels,

268 + hidden_size=self.hidden_size,

269 + )

270 +

271 + self.pos_embed = nn.Embedding(self.num_position_embeddings,

272 + self.hidden_size)

273 +

274 + norm_layer = partial(nn.LayerNorm, eps=norm_eps)

275 + head_dim = self.hidden_size // self.num_heads

276 + self.rotary_pos_emb = Qwen2_5_VisionRotaryEmbedding(head_dim // 2)

277 +

278 + self.blocks = nn.ModuleList([

279 + Qwen3_VisionBlock(

280 + dim=self.hidden_size,

281 + num_heads=self.num_heads,

282 + mlp_hidden_dim=vision_config.intermediate_size,

283 + act_fn=_ACTIVATION_REGISTRY[vision_config.hidden_act],

284 + norm_layer=norm_layer,

285 + quant_config=quant_config,

286 + prefix=f"{prefix}.blocks.{layer_idx}")

287 + for layer_idx in range(vision_config.depth)

288 + ])

289 +

290 + self.merger = Qwen3_VisionPatchMerger(

291 + d_model=vision_config.out_hidden_size,

292 + context_dim=self.hidden_size,

293 + norm_layer=norm_layer,

294 + spatial_merge_size=self.spatial_merge_size,

295 + quant_config=quant_config,

296 + prefix=f"{prefix}.merger",

297 + )

298 +

299 + self.deepstack_merger_list = nn.ModuleList([

300 + Qwen3_VisionPatchMerger(

301 + d_model=vision_config.out_hidden_size,

302 + context_dim=self.hidden_size,

303 + spatial_merge_size=self.spatial_merge_size,

304 + use_postshuffle_norm=True,

305 + norm_layer=norm_layer,

306 + quant_config=quant_config,

307 + prefix=f"{prefix}.deepstack_merger_list.{layer_idx}")

308 + for layer_idx in range(len(self.deepstack_visual_indexes))

309 + ])

310 +

311 + self.attn_backend = get_vit_attn_backend(

312 + head_size=head_dim, dtype=torch.get_default_dtype())

313 + if self.attn_backend != _Backend.FLASH_ATTN and \

314 + check_upstream_fa_availability(

315 + torch.get_default_dtype()):

316 + self.attn_backend = _Backend.FLASH_ATTN

317 +

318 + @property

319 + def dtype(self) -> torch.dtype:

320 + return self.patch_embed.proj.weight.dtype

321 +

322 + @property

323 + def device(self) -> torch.device:

324 + return self.patch_embed.proj.weight.device

325 +

326 + def rot_pos_emb(self, grid_thw):

327 + pos_ids = []

328 + for t, h, w in grid_thw:

329 + hpos_ids = torch.arange(h).unsqueeze(1).expand(-1, w)

330 + hpos_ids = hpos_ids.reshape(

331 + h // self.spatial_merge_size,

332 + self.spatial_merge_size,

333 + w // self.spatial_merge_size,

334 + self.spatial_merge_size,

335 + )

336 + hpos_ids = hpos_ids.permute(0, 2, 1, 3)

337 + hpos_ids = hpos_ids.flatten()

338 +

339 + wpos_ids = torch.arange(w).unsqueeze(0).expand(h, -1)

340 + wpos_ids = wpos_ids.reshape(

341 + h // self.spatial_merge_size,

342 + self.spatial_merge_size,

343 + w // self.spatial_merge_size,

344 + self.spatial_merge_size,

345 + )

346 + wpos_ids = wpos_ids.permute(0, 2, 1, 3)

347 + wpos_ids = wpos_ids.flatten()

348 + pos_ids.append(

349 + torch.stack([hpos_ids, wpos_ids], dim=-1).repeat(t, 1))

350 + pos_ids = torch.cat(pos_ids, dim=0)

351 + max_grid_size = grid_thw[:, 1:].max()

352 + rotary_pos_emb_full = self.rotary_pos_emb(max_grid_size)

353 + rotary_pos_emb = rotary_pos_emb_full[pos_ids].flatten(1)

354 + return rotary_pos_emb

355 +

356 + def fast_pos_embed_interpolate(self, grid_thw):

357 + num_grid_per_side = int(self.num_position_embeddings**0.5)

358 +

359 + idx_list = [[] for _ in range(4)]

360 + weight_list = [[] for _ in range(4)]

361 +

362 + for t, h, w in grid_thw:

363 + h_idxs = torch.linspace(0,

364 + num_grid_per_side - 1,

365 + h,

366 + dtype=torch.float32)

367 + w_idxs = torch.linspace(0,

368 + num_grid_per_side - 1,

369 + w,

370 + dtype=torch.float32)

371 +

372 + h_idxs_floor = h_idxs.to(torch.long)

373 + w_idxs_floor = w_idxs.to(torch.long)

374 + h_idxs_ceil = torch.clamp(h_idxs.to(torch.long) + 1,

375 + max=num_grid_per_side - 1)

376 + w_idxs_ceil = torch.clamp(w_idxs.to(torch.long) + 1,

377 + max=num_grid_per_side - 1)

378 +

379 + dh = h_idxs - h_idxs_floor

380 + dw = w_idxs - w_idxs_floor

381 +

382 + idx_list[0].extend(((h_idxs_floor * num_grid_per_side)[None].T +

383 + w_idxs_floor[None]).flatten().tolist() * t)

384 + idx_list[1].extend(((h_idxs_floor * num_grid_per_side)[None].T +

385 + w_idxs_ceil[None]).flatten().tolist() * t)

386 + idx_list[2].extend(((h_idxs_ceil * num_grid_per_side)[None].T +

387 + w_idxs_floor[None]).flatten().tolist() * t)

388 + idx_list[3].extend(((h_idxs_ceil * num_grid_per_side)[None].T +

389 + w_idxs_ceil[None]).flatten().tolist() * t)

390 +

391 + weight_list[0].extend(

392 + ((1 - dh)[None].T * (1 - dw)[None]).flatten().tolist() * t)

393 + weight_list[1].extend(

394 + ((1 - dh)[None].T * dw[None]).flatten().tolist() * t)

395 + weight_list[2].extend(

396 + (dh[None].T * (1 - dw)[None]).flatten().tolist() * t)

397 + weight_list[3].extend(

398 + (dh[None].T * dw[None]).flatten().tolist() * t)

399 +

400 + device = self.pos_embed.weight.device

401 + dtype = self.pos_embed.weight.dtype

402 +

403 + p0 = self.pos_embed(

404 + torch.tensor(

405 + idx_list[0], dtype=torch.long, device=device)) * torch.tensor(

406 + weight_list[0], dtype=dtype, device=device)[:, None]

407 + p1 = self.pos_embed(

408 + torch.tensor(

409 + idx_list[1], dtype=torch.long, device=device)) * torch.tensor(

410 + weight_list[1], dtype=dtype, device=device)[:, None]

411 + p2 = self.pos_embed(

412 + torch.tensor(

413 + idx_list[2], dtype=torch.long, device=device)) * torch.tensor(

414 + weight_list[2], dtype=dtype, device=device)[:, None]

415 + p3 = self.pos_embed(

416 + torch.tensor(

417 + idx_list[3], dtype=torch.long, device=device)) * torch.tensor(

418 + weight_list[3], dtype=dtype, device=device)[:, None]

419 +

420 + patch_pos_embeds = p0 + p1 + p2 + p3

421 + patch_pos_embeds = patch_pos_embeds.split(

422 + [t * h * w for t, h, w in grid_thw])

423 + patch_pos_embeds_permute = []

424 + m_size = self.spatial_merge_size

425 + for pos_embed, (t, h, w) in zip(patch_pos_embeds, grid_thw):

426 + pos_embed = pos_embed.view(t, h // m_size, m_size, w // m_size,

427 + m_size, -1).permute(0, 1, 3, 2, 4,

428 + 5).flatten(0, 4)

429 + patch_pos_embeds_permute.append(pos_embed)

430 + patch_pos_embeds = torch.cat(patch_pos_embeds_permute)

431 + return patch_pos_embeds

432 +

433 + def compute_attn_mask_seqlen(

434 + self,

435 + cu_seqlens: torch.Tensor,

436 + ) -> tuple[Optional[int], Optional[list[int]]]:

437 + max_seqlen, seqlens = None, None

438 + if self.attn_backend == _Backend.FLASH_ATTN:

439 + max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max().item()

440 + elif self.attn_backend == _Backend.XFORMERS:

441 + seqlens = (cu_seqlens[1:] - cu_seqlens[:-1]).tolist()

442 + return max_seqlen, seqlens

443 +

444 + def forward(

445 + self,

446 + x: torch.Tensor,

447 + grid_thw: list[list[int]],

448 + ) -> torch.Tensor:

449 + hidden_states = x.to(device=self.device, dtype=self.dtype)

450 + hidden_states = self.patch_embed(hidden_states)

451 +

452 + pos_embeds = self.fast_pos_embed_interpolate(grid_thw)

453 + hidden_states = hidden_states + pos_embeds

454 + rotary_pos_emb = self.rot_pos_emb(grid_thw)

455 +

456 + cu_seqlens = torch.repeat_interleave(

457 + grid_thw[:, 1] * grid_thw[:, 2], grid_thw[:, 0]).cumsum(

458 + dim=0,

459 + dtype=grid_thw.dtype

460 + if torch.jit.is_tracing() else torch.int32,

461 + )

462 + cu_seqlens = F.pad(cu_seqlens, (1, 0), value=0)

463 +

464 + hidden_states = hidden_states.unsqueeze(1)

465 + rotary_pos_emb = rotary_pos_emb.to(hidden_states.device)

466 + max_seqlen, seqlens = self.compute_attn_mask_seqlen(cu_seqlens)

467 +

468 + deepstack_feature_lists = []

469 + for layer_num, blk in enumerate(self.blocks):

470 + hidden_states = blk(hidden_states,

471 + cu_seqlens=cu_seqlens,

472 + rotary_pos_emb=rotary_pos_emb,

473 + max_seqlen=max_seqlen,

474 + seqlens=seqlens)

475 + if layer_num in self.deepstack_visual_indexes:

476 + deepstack_merger_idx = self.deepstack_visual_indexes.index(

477 + layer_num)

478 + deepstack_feature = self.deepstack_merger_list[

479 + deepstack_merger_idx](hidden_states)

480 + deepstack_feature_lists.append(deepstack_feature)

481 + hidden_states = self.merger(hidden_states)

482 + hidden_states = torch.cat(

483 + [hidden_states] + deepstack_feature_lists,

484 + dim=1) # [seq_len, hidden_size * (1 + depth_of_deepstack)]

485 + return hidden_states

486 +

487 + def load_weights(self, weights: Iterable[tuple[str,

488 + torch.Tensor]]) -> set[str]:

489 + stacked_params_mapping = [

490 + # (param_name, shard_name, shard_id)

491 + ("attn.qkv.", "attn.q.", "q"),

492 + ("attn.qkv.", "attn.k.", "k"),

493 + ("attn.qkv.", "attn.v.", "v"),

494 + ]

495 + params_dict = dict(self.named_parameters(remove_duplicate=False))

496 + loaded_params: set[str] = set()

497 +

498 + for name, loaded_weight in weights:

499 + for (param_name, weight_name, shard_id) in stacked_params_mapping:

500 + if weight_name not in name:

501 + continue

502 + name = name.replace(weight_name, param_name)

503 +

504 + param = params_dict[name]

505 + weight_loader = param.weight_loader

506 + weight_loader(param, loaded_weight, shard_id)

507 + break

508 + else:

509 + param = params_dict[name]

510 + weight_loader = getattr(param, "weight_loader",

511 + default_weight_loader)

512 + weight_loader(param, loaded_weight)

513 + loaded_params.add(name)

514 + return loaded_params

515 +

516 +

517 +class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):

518 +

519 + def get_hf_config(self):

520 + return self.ctx.get_hf_config(Qwen3VLConfig)

521 +

522 + def get_hf_processor(self, **kwargs: object) -> Qwen3VLProcessor:

523 + return self.ctx.get_hf_processor(

524 + Qwen3VLProcessor,

525 + use_fast=kwargs.pop("use_fast", True),

526 + **kwargs,

527 + )

528 +

529 + def get_tokenizer(self):

530 + return self.ctx.tokenizer

531 +

532 + def get_image_processor(self,

533 + **kwargs: object) -> Qwen2VLImageProcessorFast:

534 + return self.get_hf_processor(**kwargs).image_processor

535 +

536 + def get_video_processor(self, **kwargs: object) -> Qwen3VLVideoProcessor:

537 + return self.get_hf_processor(**kwargs).video_processor

538 +

539 + def _get_vision_info(

540 + self,

541 + *,

542 + image_width: int,

543 + image_height: int,

544 + num_frames: int = 2,

545 + do_resize: bool = True,

546 + image_processor: Optional[Qwen2VLImageProcessorFast],

547 + ) -> tuple[ImageSize, int]:

548 + if image_processor is None:

549 + image_processor = self.get_image_processor()

550 +

551 + hf_config = self.get_hf_config()

552 + vision_config = hf_config.vision_config

553 + patch_size = vision_config.patch_size

554 + merge_size = vision_config.spatial_merge_size

555 + temporal_patch_size = vision_config.temporal_patch_size

556 +

557 + if do_resize:

558 + resized_height, resized_width = smart_resize(

559 + height=image_height,

560 + width=image_width,

561 + factor=patch_size * merge_size,

562 + min_pixels=image_processor.size["shortest_edge"],

563 + max_pixels=image_processor.size["longest_edge"],

564 + )

565 + preprocessed_size = ImageSize(width=resized_width,

566 + height=resized_height)

567 + else:

568 + preprocessed_size = ImageSize(width=image_width,

569 + height=image_height)

570 +

571 + padded_num_frames = num_frames + num_frames % temporal_patch_size

572 +

573 + grid_t = max(padded_num_frames // temporal_patch_size, 1)

574 + grid_h = preprocessed_size.height // patch_size

575 + grid_w = preprocessed_size.width // patch_size

576 +

577 + num_patches = grid_t * grid_h * grid_w

578 + num_vision_tokens = num_patches // (merge_size**2)

579 +

580 + return preprocessed_size, num_vision_tokens

581 +

582 + def _calculate_timestamps(self, indices: list[int] | torch.Tensor,

583 + video_fps: float, merge_size: int):

584 + if not isinstance(indices, list):

585 + indices = indices.tolist()

586 + if len(indices) % merge_size != 0:

587 + # don't update metadata's frames_indices directly

588 + indices = indices + [indices[-1]

589 + ] * (merge_size - len(indices) % merge_size)

590 + timestamps = [idx / video_fps for idx in indices]

591 + timestamps = [(timestamps[i] + timestamps[i + merge_size - 1]) / 2

592 + for i in range(0, len(timestamps), merge_size)]

593 + return timestamps

594 +

595 + def _get_video_second_idx(

596 + self,

597 + metadata: dict[str, Any],

598 + out_item: MultiModalKwargsItem,

599 + do_sample_frames: Optional[bool] = None,

600 + sampled_fps: Optional[float] = None) -> list[int]:

601 + video_processor = self.get_video_processor()

602 + merge_size = video_processor.merge_size

603 + indices = metadata["frames_indices"]

604 +

605 + # metadata["fps"] refers to the true fps of the input video.

606 + video_fps = metadata["fps"]

607 + if do_sample_frames is None:

608 + do_sample_frames = metadata.get("do_sample_frames", False)

609 +

610 + # If video frames are sampled in HF processor (instead of vLLM

611 + # video loader), we need to re-calculate the indices from original

612 + # metadata.

613 + if do_sample_frames:

614 + # here video_fps is the fps of the sampled video, and

615 + # metadata["fps"] refers to the fps of the original video.

616 + video_fps = sampled_fps if sampled_fps else video_processor.fps

617 + total_num_frames = metadata["total_num_frames"]

618 + num_frames = int(total_num_frames / metadata["fps"] * video_fps)

619 + num_frames = min(

620 + min(max(num_frames, video_processor.min_frames),

621 + video_processor.max_frames), total_num_frames)

622 + indices = np.linspace(0, total_num_frames - 1,

623 + num_frames).round().astype(int).tolist()

624 + timestamps = self._calculate_timestamps(indices, video_fps, merge_size)

625 + return timestamps

626 +

627 +

628 +class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]):

629 +

630 + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:

631 + num_images = mm_counts.get("image", 0)

632 + num_videos = mm_counts.get("video", 0)

633 +

634 + image_token = "<|vision_start|><|image_pad|><|vision_end|>"

635 + video_token = "<|vision_start|><|video_pad|><|vision_end|>"

636 +

637 + return image_token * num_images + video_token * num_videos

638 +

639 + def get_dummy_mm_data(

640 + self,

641 + seq_len: int,

642 + mm_counts: Mapping[str, int],

643 + ) -> MultiModalDataDict:

644 + num_images = mm_counts.get("image", 0)

645 + num_videos = mm_counts.get("video", 0)

646 +

647 + target_width, target_height = (

648 + self.info.get_image_size_with_most_features())

649 + target_num_frames = self.info.get_num_frames_with_most_features(

650 + seq_len, mm_counts)

651 + return {

652 + "image":

653 + self._get_dummy_images(width=target_width,

654 + height=target_height,

655 + num_images=num_images),

656 + "video":

657 + self._get_dummy_videos(

658 + width=target_width,

659 + height=target_height,

660 + num_frames=target_num_frames,

661 + num_videos=num_videos,

662 + ),

663 + }

664 +

665 + def _get_dummy_videos(

666 + self,

667 + *,

668 + width: int,

669 + height: int,

670 + num_frames: int,

671 + num_videos: int,

672 + ) -> list[VideoItem]:

673 + num_frames = max(num_frames, 2)

674 + video = np.full((num_frames, width, height, 3), 255, dtype=np.uint8)

675 + video_items = []

676 + for i in range(num_videos):

677 + video_metadata = {

678 + "fps": 2.0,

679 + "duration": num_frames / 2.0,

680 + "total_num_frames": num_frames,

681 + "frames_indices": [i for i in range(num_frames)],

682 + "video_backend": "opencv",

683 + "do_sample_frames": False,

684 + }

685 + video_item = (video.copy(), video_metadata)

686 + video_items.append(video_item)

687 + return video_items

688 +

689 +

690 +class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]

691 + ):

692 +

693 + def _get_data_parser(self) -> MultiModalDataParser:

694 + return MultiModalDataParser(video_needs_metadata=True)

695 +

696 + def _call_hf_processor(

697 + self,

698 + prompt: str,

699 + mm_data: Mapping[str, object],

700 + mm_kwargs: Mapping[str, object],

701 + tok_kwargs: Mapping[str, object],

702 + ) -> BatchFeature:

703 + mm_data = dict(mm_data)

704 + processor = self.info.get_hf_processor(**mm_kwargs)

705 +

706 + # Separate video processing from image processing. Because the videos

707 + # are processed into serval image patches

708 + if ("videos" in mm_data and isinstance(mm_data["videos"], list)

709 + and len(mm_data["videos"]) > 0):

710 + video_grid_thw_lst = []

711 + pixel_values_videos_lst = []

712 +

713 + for item_idx, item in enumerate(mm_data.pop("videos", [])):

714 + video_array, metadata = item

715 +

716 + # NOTE: @JJJYmmm new attr metadata.frames_indices indicates

717 + # the sampled frames indices of pre-sampled videos, which is

718 + # used to calculate the timestamps. Make sure that

719 + # do_sample_frames in mm_kwargs is false for presampled videos.

720 +

721 + # NOTE: a copy of is created to update do_sample_frames,

722 + # otherwise mm_hash for the object will be incorrect.

723 + video_mm_kwargs = dict(**mm_kwargs)

724 + if "do_sample_frames" not in video_mm_kwargs:

725 + # qwen_vl_utils already has "do_sample_frames" in

726 + # mm_kwargs, don't overwrite it.

727 + video_mm_kwargs["do_sample_frames"] = metadata.get(

728 + "do_sample_frames", False)

729 +

730 + metadata = VideoMetadata(**{

731 + k: metadata[k]

732 + for k in metadata if k != "do_sample_frames"

733 + })

734 +

735 + video_mm_data = dict()

736 + video_mm_data["videos"] = [[video_array]]

737 + video_mm_data["video_metadata"] = [[metadata]]

738 +

739 + video_outputs = super()._call_hf_processor(

740 + prompt="<|vision_start|><|video_pad|><|vision_end|>",

741 + mm_data=video_mm_data,

742 + mm_kwargs=video_mm_kwargs,

743 + tok_kwargs=tok_kwargs,

744 + )

745 + input_ids = video_outputs.pop("input_ids")

746 + video_placeholder = processor.tokenizer.batch_decode(

747 + input_ids)[0]

748 + prompt = prompt.replace(

749 + "<|vision_start|><|video_pad|><|vision_end|>",

750 + video_placeholder,

751 + 1,

752 + )

753 +

754 + video_grid_thw_lst.append(video_outputs["video_grid_thw"])

755 + pixel_values_videos_lst.append(

756 + video_outputs["pixel_values_videos"])

757 + video_outputs = dict(

758 + pixel_values_videos=torch.cat(pixel_values_videos_lst),

759 + video_grid_thw=torch.cat(video_grid_thw_lst),

760 + )

761 + else:

762 + video_outputs = dict()

763 +

764 + processed_outputs = super()._call_hf_processor(

765 + prompt=prompt,

766 + mm_data=mm_data,

767 + mm_kwargs=mm_kwargs,

768 + tok_kwargs=tok_kwargs,

769 + )

770 + combined_outputs = dict(

771 + processed_outputs,

772 + **video_outputs,

773 + )

774 + return BatchFeature(combined_outputs)

775 +

776 + def _get_mm_fields_config(

777 + self,

778 + hf_inputs: BatchFeature,

779 + hf_processor_mm_kwargs: Mapping[str, object],

780 + ) -> Mapping[str, MultiModalFieldConfig]:

781 + image_grid_thw = hf_inputs.get("image_grid_thw", torch.empty((0, 3)))

782 + image_grid_sizes = image_grid_thw.prod(-1)

783 +

784 + video_grid_thw = hf_inputs.get("video_grid_thw", torch.empty((0, 3)))

785 + video_grid_sizes = video_grid_thw.prod(-1)

786 +

787 + return dict(

788 + pixel_values=MultiModalFieldConfig.flat_from_sizes(

789 + "image", image_grid_sizes),

790 + image_embeds=MultiModalFieldConfig.flat_from_sizes(

791 + "image", image_grid_sizes),

792 + image_grid_thw=MultiModalFieldConfig.batched("image"),

793 + pixel_values_videos=MultiModalFieldConfig.flat_from_sizes(

794 + "video", video_grid_sizes),

795 + video_embeds=MultiModalFieldConfig.flat_from_sizes(

796 + "video", video_grid_sizes),

797 + video_grid_thw=MultiModalFieldConfig.batched("video"),

798 + )

799 +

800 + def _get_prompt_updates(

801 + self,

802 + mm_items: MultiModalDataItems,

803 + hf_processor_mm_kwargs: Mapping[str, Any],

804 + out_mm_kwargs: MultiModalKwargsItems,

805 + ) -> Sequence[PromptUpdate]:

806 + hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)

807 + image_processor = self.info.get_image_processor(

808 + **hf_processor_mm_kwargs)

809 + tokenizer = self.info.get_tokenizer()

810 + hf_config = self.info.get_hf_config()

811 +

812 + video_token_id = hf_config.video_token_id

813 + vision_start_token_id = hf_config.vision_start_token_id

814 + vision_end_token_id = hf_config.vision_end_token_id

815 +

816 + merge_length = image_processor.merge_size**2

817 +

818 + def get_image_replacement_qwen3vl(item_idx: int):

819 + out_item = out_mm_kwargs["image"][item_idx]

820 + grid_thw = out_item["image_grid_thw"].data

821 + assert isinstance(grid_thw, torch.Tensor)

822 +

823 + num_tokens = int(grid_thw.prod()) // merge_length

824 + return [hf_processor.image_token_id] * num_tokens

825 +

826 + def get_video_replacement_qwen3vl(item_idx: int):

827 + out_item = out_mm_kwargs["video"][item_idx]

828 + grid_thw = out_item["video_grid_thw"].data

829 + assert isinstance(grid_thw, torch.Tensor)

830 +

831 + video, metadata = mm_items["video"][item_idx]

832 + do_sample_frames = hf_processor_mm_kwargs.get("do_sample_frames")

833 + sampled_fps = hf_processor_mm_kwargs.get("fps")

834 + if is_list_of(sampled_fps, float):

835 + sampled_fps = sampled_fps[item_idx]

836 + timestamps = self.info._get_video_second_idx(

837 + metadata, out_item, do_sample_frames, sampled_fps)

838 +

839 + assert len(timestamps) == grid_thw[0], (

840 + f"The timestamps length({len(timestamps)}) should be equal "

841 + f"video length ({grid_thw[0]}).")

842 +

843 + frames_idx_token = [

844 + tokenizer.encode(f"<{curr_time:.1f} seconds>",

845 + add_special_tokens=False)

846 + for curr_time in timestamps

847 + ]

848 + num_tokens_per_frame = int(grid_thw[1:].prod()) // merge_length

849 + placeholder = []

850 + for frame_idx in frames_idx_token:

851 + placeholder.extend(frame_idx)

852 + placeholder.extend([vision_start_token_id] +

853 + [video_token_id] * num_tokens_per_frame +

854 + [vision_end_token_id])

855 + return PromptUpdateDetails.select_token_id(placeholder,

856 + video_token_id)

857 +

858 + return [

859 + PromptReplacement(

860 + modality="image",

861 + target=hf_processor.image_token,

862 + replacement=get_image_replacement_qwen3vl,

863 + ),

864 +

865 + # NOTE: We match string on purpose since searching sequence of

866 + # token ids takes more time.

867 + PromptReplacement(

868 + modality="video",

869 + target="<|vision_start|><|video_pad|><|vision_end|>",

870 + replacement=get_video_replacement_qwen3vl,

871 + ),

872 + ]

873 +

874 +

875 +@support_torch_compile(

876 + dynamic_arg_dims={

877 + "input_ids": 0,

878 + # positions is of shape (3, seq_len) if mrope is enabled for qwen2-vl,

879 + # otherwise (seq_len, ).

880 + "positions": -1,

881 + "intermediate_tensors": 0,

882 + "inputs_embeds": 0,

883 + # the same shape as input_embeds

884 + "deepstack_input_embeds": 0

885 + })

886 +class Qwen3LLMModel(Qwen3Model):

887 +

888 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):

889 + super().__init__(vllm_config=vllm_config, prefix=prefix)

890 + if not get_pp_group().is_first_rank:

891 + assert self.start_layer >= len(

892 + vllm_config.model_config.hf_config.vision_config.

893 + deepstack_visual_indexes), (

894 + "start_layer should be greater than or equal to "

895 + "len(deepstack_visual_indexes)")

896 +

897 + def forward(

898 + self,

899 + input_ids: torch.Tensor,

900 + positions: torch.Tensor,

901 + intermediate_tensors: Optional[IntermediateTensors] = None,

902 + inputs_embeds: Optional[torch.Tensor] = None,

903 + # args for deepstack

904 + deepstack_input_embeds: Optional[IntermediateTensors] = None,

905 + ) -> Union[torch.Tensor, IntermediateTensors]:

906 + if get_pp_group().is_first_rank:

907 + if inputs_embeds is not None:

908 + hidden_states = inputs_embeds

909 + else:

910 + hidden_states = self.get_input_embeddings(input_ids)

911 + residual = None

912 + else:

913 + assert intermediate_tensors is not None

914 + hidden_states = intermediate_tensors["hidden_states"]

915 + residual = intermediate_tensors["residual"]

916 + for layer_idx, layer in enumerate(

917 + self.layers[self.start_layer:self.end_layer]):

918 + layer_idx = layer_idx + self.start_layer

919 +

920 + hidden_states, residual = layer(

921 + positions,

922 + hidden_states,

923 + residual,

924 + )

925 +

926 + if deepstack_input_embeds is not None and \

927 + layer_idx in range(0, len(deepstack_input_embeds)):

928 + hidden_states = hidden_states + deepstack_input_embeds[

929 + f"deepstack_input_embeds_{layer_idx}"]

930 +

931 + if not get_pp_group().is_last_rank:

932 + return IntermediateTensors({

933 + "hidden_states": hidden_states,

934 + "residual": residual

935 + })

936 + hidden_states, _ = self.norm(hidden_states, residual)

937 + return hidden_states

938 +

939 +

940 +class Qwen3LLMForCausalLM(Qwen3ForCausalLM):

941 +

942 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):

943 + super(Qwen3ForCausalLM, self).__init__()

944 + config = vllm_config.model_config.hf_config.text_config

945 + quant_config = vllm_config.quant_config

946 + lora_config = vllm_config.lora_config

947 +

948 + self.config = config

949 + self.lora_config = lora_config

950 +

951 + self.quant_config = quant_config

952 + self.model = Qwen3LLMModel(vllm_config=vllm_config, prefix=prefix)

953 +

954 + if get_pp_group().is_last_rank:

955 + if config.tie_word_embeddings:

956 + self.lm_head = self.model.embed_tokens

957 + else:

958 + self.lm_head = ParallelLMHead(config.vocab_size,

959 + config.hidden_size,

960 + quant_config=quant_config,

961 + prefix="lm_head")

962 + else:

963 + self.lm_head = PPMissingLayer()

964 +

965 + self.logits_processor = LogitsProcessor(config.vocab_size)

966 +

967 + self.make_empty_intermediate_tensors = (

968 + self.model.make_empty_intermediate_tensors)

969 +

970 +

971 +@MULTIMODAL_REGISTRY.register_processor(Qwen3VLMultiModalProcessor,

972 + info=Qwen3VLProcessingInfo,

973 + dummy_inputs=Qwen3VLDummyInputsBuilder)

974 +class Qwen3VLForConditionalGeneration(nn.Module, SupportsMultiModal,

975 + SupportsLoRA, SupportsPP):

976 + packed_modules_mapping = {

977 + "qkv_proj": [

978 + "q_proj",

979 + "k_proj",

980 + "v_proj",

981 + ],

982 + "gate_up_proj": [

983 + "gate_proj",

984 + "up_proj",

985 + ],

986 + }

987 + # To ensure correct weight loading and mapping.

988 + hf_to_vllm_mapper = WeightsMapper(

989 + orig_to_new_prefix={

990 + "model.visual.": "visual.",

991 + "lm_head.": "language_model.lm_head.",

992 + "model.language_model.": "language_model.model.",

993 + })

994 +

995 + @classmethod

996 + def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:

997 + if modality.startswith("image"):

998 + return "<|vision_start|><|image_pad|><|vision_end|>"

999 + if modality.startswith("video"):

1000 + return "<|vision_start|><|video_pad|><|vision_end|>"

1001 +

1002 + raise ValueError("Only image or video modality is supported")

1003 +

1004 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = "model"):

1005 + super().__init__()

1006 + config: Qwen3VLConfig = vllm_config.model_config.hf_config

1007 + quant_config = vllm_config.quant_config

1008 + multimodal_config = vllm_config.model_config.multimodal_config

1009 +

1010 + self.config = config

1011 + self.multimodal_config = multimodal_config

1012 +

1013 + self.visual = Qwen3_VisionTransformer(

1014 + config.vision_config,

1015 + norm_eps=getattr(config, "rms_norm_eps", 1e-6),

1016 + quant_config=self._maybe_ignore_quant_config(quant_config),

1017 + prefix=maybe_prefix(prefix, "visual"),

1018 + )

1019 +

1020 + self.language_model = Qwen3LLMForCausalLM(vllm_config=vllm_config,

1021 + prefix=maybe_prefix(

1022 + prefix,

1023 + "language_model"))

1024 +

1025 + self.make_empty_intermediate_tensors = (

1026 + self.language_model.make_empty_intermediate_tensors)

1027 +

1028 + self.use_deepstack = hasattr(config.vision_config,

1029 + 'deepstack_visual_indexes')

1030 + self.deepstack_num_level = len(

1031 + config.vision_config.deepstack_visual_indexes

1032 + ) if self.use_deepstack else 0

1033 + # register buffer for deepstack

1034 + self.deepstack_input_embeds = [

1035 + torch.zeros(vllm_config.scheduler_config.max_num_batched_tokens,

1036 + config.text_config.hidden_size)

1037 + for _ in range(self.deepstack_num_level)

1038 + ] if self.use_deepstack else None

1039 +

1040 + def _get_deepstack_input_embeds(self,

1041 + num_tokens: int) -> IntermediateTensors:

1042 + # get deepstack_input_embeds from buffer, and clear the buffer

1043 + return IntermediateTensors({

1044 + f"deepstack_input_embeds_{idx}":

1045 + self.deepstack_input_embeds[idx][:num_tokens]

1046 + for idx in range(self.deepstack_num_level)

1047 + })

1048 +

1049 + def _set_deepstack_input_embeds(

1050 + self, deepstack_input_embeds: torch.Tensor) -> None:

1051 + # set deepstack_input_embeds to buffer

1052 + num_tokens = deepstack_input_embeds.size(1)

1053 + if num_tokens > self.deepstack_input_embeds[0].size(0):

1054 + self.deepstack_input_embeds = [

1055 + torch.zeros(num_tokens,

1056 + self.config.text_config.hidden_size,

1057 + device=self.deepstack_input_embeds[0].device,

1058 + dtype=self.deepstack_input_embeds[0].dtype)

1059 + for _ in range(self.deepstack_num_level)

1060 + ]

1061 + for idx in range(self.deepstack_num_level):

1062 + self.deepstack_input_embeds[idx][:num_tokens].copy_(

1063 + deepstack_input_embeds[idx])

1064 +

1065 + def _clear_deepstack_input_embeds(self, num_tokens: int) -> None:

1066 + # clear deepstack_input_embeds in buffer

1067 + if num_tokens > 0:

1068 + for idx in range(self.deepstack_num_level):

1069 + self.deepstack_input_embeds[idx][:num_tokens].zero_()

1070 +

1071 + def _maybe_ignore_quant_config(self, quant_config: QuantizationConfig):

1072 + # GPTQ configs do not have a list of ignored modules, however AutoGPTQ

1073 + # seems to avoid vision encoder sections for some models.

1074 + if isinstance(quant_config, (GPTQConfig, GPTQMarlinConfig)):

1075 + return None

1076 + return quant_config

1077 +

1078 + def _validate_and_reshape_mm_tensor(self, mm_input: object,

1079 + name: str) -> torch.Tensor:

1080 + if not isinstance(mm_input, (torch.Tensor, list)):

1081 + raise ValueError(f"Incorrect type of {name}. "

1082 + f"Got type: {type(mm_input)}")

1083 + if isinstance(mm_input, torch.Tensor):

1084 + if mm_input.ndim == 2:

1085 + return mm_input

1086 + if mm_input.ndim != 3:

1087 + raise ValueError(f"{name} should be 2D or batched 3D tensor. "

1088 + f"Got ndim: {mm_input.ndim} "

1089 + f"(shape={mm_input.shape})")

1090 + return torch.concat(list(mm_input))

1091 + else:

1092 + return torch.concat(mm_input)

1093 +

1094 + def _parse_and_validate_image_input(

1095 + self, **kwargs: object) -> Optional[Qwen2_5_VLImageInputs]:

1096 + pixel_values = kwargs.pop("pixel_values", None)

1097 + image_embeds = kwargs.pop("image_embeds", None)

1098 + image_grid_thw = kwargs.pop("image_grid_thw", None)

1099 +

1100 + if pixel_values is None and image_embeds is None:

1101 + return None

1102 +

1103 + if pixel_values is not None:

1104 + pixel_values = self._validate_and_reshape_mm_tensor(

1105 + pixel_values, "image pixel values")

1106 + image_grid_thw = self._validate_and_reshape_mm_tensor(

1107 + image_grid_thw, "image grid_thw")

1108 +

1109 + if not isinstance(pixel_values, (torch.Tensor, list)):

1110 + raise ValueError("Incorrect type of image pixel values. "

1111 + f"Got type: {type(pixel_values)}")

1112 +

1113 + return Qwen2_5_VLImagePixelInputs(type="pixel_values",

1114 + pixel_values=pixel_values,

1115 + image_grid_thw=image_grid_thw)

1116 +

1117 + if image_embeds is not None:

1118 + image_embeds = self._validate_and_reshape_mm_tensor(

1119 + image_embeds, "image embeds")

1120 + image_grid_thw = self._validate_and_reshape_mm_tensor(

1121 + image_grid_thw, "image grid_thw")

1122 +

1123 + if not isinstance(image_embeds, torch.Tensor):

1124 + raise ValueError("Incorrect type of image embeddings. "

1125 + f"Got type: {type(image_embeds)}")

1126 + return Qwen2_5_VLImageEmbeddingInputs(

1127 + type="image_embeds",

1128 + image_embeds=image_embeds,

1129 + image_grid_thw=image_grid_thw)

1130 +

1131 + def _parse_and_validate_video_input(

1132 + self, **kwargs: object) -> Optional[Qwen2_5_VLVideoInputs]:

1133 + pixel_values_videos = kwargs.pop("pixel_values_videos", None)

1134 + video_embeds = kwargs.pop("video_embeds", None)

1135 + video_grid_thw = kwargs.pop("video_grid_thw", None)

1136 + second_per_grid_ts = kwargs.pop("second_per_grid_ts", None)

1137 +

1138 + if pixel_values_videos is None and video_embeds is None:

1139 + return None

1140 +

1141 + if pixel_values_videos is not None:

1142 + pixel_values_videos = self._validate_and_reshape_mm_tensor(

1143 + pixel_values_videos, "video pixel values")

1144 + video_grid_thw = self._validate_and_reshape_mm_tensor(

1145 + video_grid_thw, "video grid_thw")

1146 +

1147 + return Qwen2_5_VLVideoPixelInputs(

1148 + type="pixel_values_videos",

1149 + pixel_values_videos=pixel_values_videos,

1150 + video_grid_thw=video_grid_thw,

1151 + second_per_grid_ts=second_per_grid_ts,

1152 + )

1153 +

1154 + if video_embeds is not None:

1155 + video_embeds = self._validate_and_reshape_mm_tensor(

1156 + video_embeds, "video embeds")

1157 + video_grid_thw = self._validate_and_reshape_mm_tensor(

1158 + video_grid_thw, "video grid_thw")

1159 +

1160 + if not isinstance(video_embeds, torch.Tensor):

1161 + raise ValueError("Incorrect type of video embeddings. "

1162 + f"Got type: {type(video_embeds)}")

1163 + return Qwen2_5_VLVideoEmbeddingInputs(

1164 + type="video_embeds",

1165 + video_embeds=video_embeds,

1166 + video_grid_thw=video_grid_thw)

1167 +

1168 + def _process_image_input(

1169 + self,

1170 + image_input: Qwen2_5_VLImageInputs) -> tuple[torch.Tensor, ...]:

1171 +

1172 + grid_thw = image_input["image_grid_thw"]

1173 + assert grid_thw.ndim == 2

1174 + grid_thw_list = grid_thw.tolist()

1175 +

1176 + if image_input["type"] == "image_embeds":

1177 + image_embeds = image_input["image_embeds"].type(self.visual.dtype)

1178 + else:

1179 + pixel_values = image_input["pixel_values"].type(self.visual.dtype)

1180 + image_embeds = self.visual(pixel_values, grid_thw=grid_thw)

1181 +

1182 + # Split concatenated embeddings for each image item.

1183 + # Using prod on grid_thw_list instead of grid_thw.prod avoids CUDA sync

1184 + merge_size = self.visual.spatial_merge_size

1185 + sizes = (torch.tensor(grid_thw_list, dtype=torch.long).prod(-1) //

1186 + (merge_size * merge_size)).tolist()

1187 + return image_embeds.split(sizes)

1188 +

1189 + def _process_video_input(

1190 + self,

1191 + video_input: Qwen2_5_VLVideoInputs) -> tuple[torch.Tensor, ...]:

1192 +

1193 + grid_thw = video_input["video_grid_thw"]

1194 + assert grid_thw.ndim == 2

1195 + grid_thw_list = grid_thw.tolist()

1196 +

1197 + if video_input["type"] == "video_embeds":

1198 + video_embeds = video_input["video_embeds"].type(self.visual.dtype)

1199 + else:

1200 + pixel_values_videos = video_input["pixel_values_videos"].type(

1201 + self.visual.dtype)

1202 + video_embeds = self.visual(pixel_values_videos, grid_thw=grid_thw)

1203 +

1204 + # Split concatenated embeddings for each video item.

1205 + # Using prod on grid_thw_list instead of grid_thw.prod avoids CUDA sync

1206 + merge_size = self.visual.spatial_merge_size

1207 + sizes = (torch.tensor(grid_thw_list, dtype=torch.long).prod(-1) //

1208 + (merge_size * merge_size)).tolist()

1209 + return video_embeds.split(sizes)

1210 +

1211 + def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:

1212 + mm_input_by_modality = {}

1213 + for input_key in kwargs:

1214 + if input_key in ("pixel_values", "image_embeds"

1215 + ) and "image" not in mm_input_by_modality:

1216 + mm_input_by_modality[

1217 + "image"] = self._parse_and_validate_image_input(**kwargs)

1218 + if input_key in ("pixel_values_videos", "video_embeds"

1219 + ) and "video" not in mm_input_by_modality:

1220 + mm_input_by_modality[

1221 + "video"] = self._parse_and_validate_video_input(**kwargs)

1222 + return mm_input_by_modality

1223 +

1224 + def get_language_model(self) -> torch.nn.Module:

1225 + return self.language_model

1226 +

1227 + def get_multimodal_embeddings(

1228 + self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

1229 +

1230 + mm_input_by_modality = self._parse_and_validate_multimodal_inputs(

1231 + **kwargs)

1232 + if not mm_input_by_modality:

1233 + return None

1234 +

1235 + # The result multimodal_embeddings is tuple of tensors, with each

1236 + # tensor correspoending to a multimodal data item (image or video).

1237 + multimodal_embeddings: tuple[torch.Tensor, ...] = ()

1238 +

1239 + # NOTE: It is important to iterate over the keys in this dictionary

1240 + # to preserve the order of the modalities.

1241 + for modality in mm_input_by_modality:

1242 + multimodal_input = mm_input_by_modality[modality]

1243 + if modality == "image":

1244 + vision_embeddings = self._process_image_input(multimodal_input)

1245 + multimodal_embeddings += vision_embeddings

1246 + if modality == "video":

1247 + video_embeddings = self._process_video_input(multimodal_input)

1248 + multimodal_embeddings += video_embeddings

1249 + return multimodal_embeddings

1250 +

1251 + def _compute_deepstack_embeds(

1252 + self, input_ids: torch.Tensor, inputs_embeds: torch.Tensor,

1253 + multimodal_embeddings: MultiModalEmbeddings) -> torch.Tensor:

1254 + visual_lens = [

1255 + x.shape[0] if isinstance(x, torch.Tensor) else len(x)

1256 + for x in multimodal_embeddings

1257 + ]

1258 + multimodal_embeddings_cat = torch.cat(multimodal_embeddings, dim=0)

1259 +

1260 + visual_dim = multimodal_embeddings_cat.shape[-1] // (

1261 + self.deepstack_num_level + 1)

1262 +

1263 + main_dim, multi_dim = visual_dim, visual_dim * self.deepstack_num_level

1264 + multimodal_embeddings_main, multimodal_embeddings_multiscale = torch.split( # noqa:E501

1265 + multimodal_embeddings_cat, [main_dim, multi_dim],

1266 + dim=-1)

1267 +

1268 + multimodal_embeddings = torch.split(multimodal_embeddings_main,

1269 + visual_lens,

1270 + dim=0)

1271 + multimodal_embeddings_multiscale = torch.split(

1272 + multimodal_embeddings_multiscale, visual_lens, dim=0)

1273 +

1274 + deepstack_input_embeds = inputs_embeds.new_zeros(

1275 + inputs_embeds.size(0),

1276 + self.deepstack_num_level * inputs_embeds.size(1))

1277 +

1278 + deepstack_input_embeds = merge_multimodal_embeddings(

1279 + input_ids,

1280 + deepstack_input_embeds,

1281 + multimodal_embeddings_multiscale,

1282 + placeholder_token_id=[

1283 + self.config.image_token_id, self.config.video_token_id

1284 + ],

1285 + )

1286 + deepstack_input_embeds = deepstack_input_embeds.view(

1287 + inputs_embeds.shape[0], self.deepstack_num_level,

1288 + visual_dim).contiguous()

1289 + deepstack_input_embeds = deepstack_input_embeds.permute(

1290 + 1, 0, 2).contiguous()

1291 + return deepstack_input_embeds, multimodal_embeddings

1292 +

1293 + def get_input_embeddings(

1294 + self,

1295 + input_ids: torch.Tensor,

1296 + multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

1297 + ) -> torch.Tensor:

1298 + deepstack_input_embeds = None

1299 + inputs_embeds = self.language_model.get_input_embeddings(input_ids)

1300 + if multimodal_embeddings is not None and self.use_deepstack:

1301 + deepstack_input_embeds, multimodal_embeddings = self._compute_deepstack_embeds( # noqa:E501

1302 + input_ids, inputs_embeds, multimodal_embeddings)

1303 + inputs_embeds = merge_multimodal_embeddings(

1304 + input_ids, inputs_embeds, multimodal_embeddings,

1305 + [self.config.image_token_id, self.config.video_token_id])

1306 +

1307 + if self.use_deepstack:

1308 + if deepstack_input_embeds is None:

1309 + deepstack_input_embeds = torch.zeros_like(

1310 + inputs_embeds).unsqueeze(0).repeat(

1311 + self.deepstack_num_level, 1, 1).contiguous()

1312 + self._set_deepstack_input_embeds(deepstack_input_embeds)

1313 +

1314 + return inputs_embeds

1315 +

1316 + def get_input_embeddings_v0(

1317 + self,

1318 + input_ids: torch.Tensor,

1319 + image_input: Optional[Qwen2_5_VLImageInputs] = None,

1320 + video_input: Optional[Qwen2_5_VLVideoInputs] = None,

1321 + ) -> torch.Tensor:

1322 + inputs_embeds = self.get_input_embeddings(input_ids)

1323 +

1324 + if self.use_deepstack:

1325 + visual_dim = inputs_embeds.shape[-1]

1326 + deepstack_input_embeds = None

1327 + if image_input is not None or video_input is not None:

1328 + deepstack_input_embeds = torch.zeros_like(

1329 + inputs_embeds).unsqueeze(1).repeat(

1330 + 1, self.deepstack_num_level, 1).flatten(1)

1331 +

1332 + if image_input is not None:

1333 + image_embeds = self._process_image_input(image_input)

1334 + if self.use_deepstack:

1335 + image_embeds = torch.cat(image_embeds)

1336 +

1337 + image_embeds, image_embeds_multiscale = image_embeds.split(

1338 + [visual_dim, visual_dim * self.deepstack_num_level],

1339 + dim=-1)

1340 +

1341 + deepstack_input_embeds = merge_multimodal_embeddings(

1342 + input_ids,

1343 + deepstack_input_embeds,

1344 + image_embeds_multiscale,

1345 + placeholder_token_id=self.config.image_token_id,

1346 + )

1347 +

1348 + inputs_embeds = merge_multimodal_embeddings(

1349 + input_ids,

1350 + inputs_embeds,

1351 + image_embeds,

1352 + placeholder_token_id=self.config.image_token_id,

1353 + )

1354 +

1355 + if video_input is not None:

1356 + video_embeds = self._process_video_input(video_input)

1357 + if self.use_deepstack:

1358 + video_embeds = torch.cat(video_embeds)

1359 +

1360 + video_embeds, video_embeds_multiscale = video_embeds.split(

1361 + [visual_dim, visual_dim * self.deepstack_num_level],

1362 + dim=-1)

1363 +

1364 + deepstack_input_embeds = merge_multimodal_embeddings(

1365 + input_ids,

1366 + deepstack_input_embeds,

1367 + video_embeds_multiscale,

1368 + placeholder_token_id=self.config.video_token_id,

1369 + )

1370 +

1371 + inputs_embeds = merge_multimodal_embeddings(

1372 + input_ids,

1373 + inputs_embeds,

1374 + video_embeds,

1375 + placeholder_token_id=self.config.video_token_id,

1376 + )

1377 +

1378 + if self.use_deepstack and deepstack_input_embeds is not None:

1379 + deepstack_input_embeds = deepstack_input_embeds.view(

1380 + inputs_embeds.shape[0], self.deepstack_num_level,

1381 + visual_dim).permute(1, 0, 2).contiguous()

1382 + self._set_deepstack_input_embeds(deepstack_input_embeds)

1383 + return inputs_embeds

1384 +

1385 + def forward(

1386 + self,

1387 + input_ids: torch.Tensor,

1388 + positions: torch.Tensor,

1389 + intermediate_tensors: Optional[IntermediateTensors] = None,

1390 + inputs_embeds: Optional[torch.Tensor] = None,

1391 + **kwargs: object,

1392 + ) -> Union[torch.Tensor, IntermediateTensors]:

1393 + """Run forward pass for Qwen3VL.

1394 +

1395 + Args:

1396 + input_ids: Flattened (concatenated) input_ids corresponding to a

1397 + batch.

1398 + positions: Flattened (concatenated) position ids corresponding to a

1399 + batch.

1400 + **NOTE**: If mrope is enabled (default setting for Qwen3VL

1401 + opensource models), the shape will be `(3, seq_len)`,

1402 + otherwise it will be `(seq_len,).

1403 + pixel_values: Pixel values to be fed to a model.

1404 + `None` if no images are passed.

1405 + image_grid_thw: Tensor `(n_images, 3)` of image 3D grid in LLM.

1406 + `None` if no images are passed.

1407 + pixel_values_videos: Pixel values of videos to be fed to a model.

1408 + `None` if no videos are passed.

1409 + video_grid_thw: Tensor `(n_videos, 3)` of video 3D grid in LLM.

1410 + `None` if no videos are passed.

1411 + """

1412 +

1413 + if intermediate_tensors is not None:

1414 + inputs_embeds = None

1415 +

1416 + # NOTE: In v1, inputs_embeds is always generated at model runner from

1417 + # `get_multimodal_embeddings` and `get_input_embeddings`, this

1418 + # condition is only for v0 compatibility.

1419 + elif inputs_embeds is None:

1420 + image_input = self._parse_and_validate_image_input(**kwargs)

1421 + video_input = self._parse_and_validate_video_input(**kwargs)

1422 +

1423 + if image_input is None and video_input is None:

1424 + inputs_embeds = None

1425 + else:

1426 + if uses_mrope(self.config):

1427 + assert positions.ndim == 2 and positions.size(0) == 3, (

1428 + "multimodal section rotary embedding requires "

1429 + f"(3, seq_len) positions, but got {positions.size()}")

1430 + inputs_embeds = self.get_input_embeddings_v0(

1431 + input_ids,

1432 + image_input=image_input,

1433 + video_input=video_input)

1434 + input_ids = None

1435 +

1436 + if self.use_deepstack and inputs_embeds is not None and get_pp_group(

1437 + ).is_first_rank:

1438 + deepstack_input_embeds = self._get_deepstack_input_embeds(

1439 + inputs_embeds.size(0))

1440 + else:

1441 + deepstack_input_embeds = None

1442 +

1443 + hidden_states = self.language_model.model(

1444 + input_ids=input_ids,

1445 + positions=positions,

1446 + intermediate_tensors=intermediate_tensors,

1447 + inputs_embeds=inputs_embeds,

1448 + # args for deepstack

1449 + deepstack_input_embeds=deepstack_input_embeds,

1450 + )

1451 +

1452 + if inputs_embeds is not None and get_pp_group().is_first_rank:

1453 + self._clear_deepstack_input_embeds(inputs_embeds.size(0))

1454 +

1455 + return hidden_states

1456 +

1457 + def compute_logits(

1458 + self,

1459 + hidden_states: torch.Tensor,

1460 + sampling_metadata: SamplingMetadata,

1461 + ) -> Optional[torch.Tensor]:

1462 + return self.language_model.compute_logits(hidden_states,

1463 + sampling_metadata)

1464 +

1465 + def load_weights(self, weights: Iterable[tuple[str,

1466 + torch.Tensor]]) -> set[str]:

1467 + loader = AutoWeightsLoader(self)

1468 + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)

1469 +

1470 + def get_mm_mapping(self) -> MultiModelKeys:

1471 + """

1472 + Get the module prefix in multimodal models

1473 + """

1474 + return MultiModelKeys.from_string_field(

1475 + language_model="language_model",

1476 + connector="model.visual.merger",

1477 + tower_model="model.visual.",

1478 + )

vllm/model_executor/models/qwen3_vl_moe.py new

+344

@@ -0,0 +1,344 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +

8 +#

9 +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX

10 +# and OPT implementations in this library. It has been modified from its

11 +# original forms to accommodate minor architectural differences compared

12 +# to GPT-NeoX and OPT used by the Meta AI team that trained the model.

13 +#

14 +# Licensed under the Apache License, Version 2.0 (the "License");

15 +# you may not use this file except in compliance with the License.

16 +# You may obtain a copy of the License at

17 +#

18 +# http://www.apache.org/licenses/LICENSE-2.0

19 +#

20 +# Unless required by applicable law or agreed to in writing, software

21 +# distributed under the License is distributed on an "AS IS" BASIS,

22 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

23 +# See the License for the specific language governing permissions and

24 +# limitations under the License.

25 +"""Inference-only Qwen3-VL-MoE model compatible with HuggingFace weights."""

26 +import typing

27 +from collections.abc import Iterable

28 +from typing import Callable, Optional, Union

29 +

30 +import torch

31 +from transformers.models.qwen3_vl_moe.configuration_qwen3_vl_moe import (

32 + Qwen3VLMoeConfig)

33 +

34 +from vllm.compilation.decorators import support_torch_compile

35 +from vllm.config import VllmConfig

36 +from vllm.distributed import get_pp_group

37 +from vllm.logger import init_logger

38 +from vllm.model_executor.layers.logits_processor import LogitsProcessor

39 +from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead

40 +from vllm.model_executor.model_loader.weight_utils import (

41 + default_weight_loader, maybe_remap_kv_scale_name)

42 +from vllm.multimodal import MULTIMODAL_REGISTRY

43 +from vllm.sequence import IntermediateTensors

44 +

45 +from .qwen3_moe import Qwen3MoeForCausalLM, Qwen3MoeModel

46 +from .qwen3_vl import (Qwen3_VisionTransformer, Qwen3VLDummyInputsBuilder,

47 + Qwen3VLForConditionalGeneration,

48 + Qwen3VLMultiModalProcessor, Qwen3VLProcessingInfo)

49 +from .utils import is_pp_missing_parameter, maybe_prefix

50 +

51 +logger = init_logger(__name__)

52 +

53 +

54 +class Qwen3VLMoeProcessingInfo(Qwen3VLProcessingInfo):

55 +

56 + def get_hf_config(self):

57 + return self.ctx.get_hf_config(Qwen3VLMoeConfig)

58 +

59 +

60 +@support_torch_compile(

61 + dynamic_arg_dims={

62 + "input_ids": 0,

63 + # positions is of shape (3, seq_len) if mrope is enabled for qwen2-vl,

64 + # otherwise (seq_len, ).

65 + "positions": -1,

66 + "intermediate_tensors": 0,

67 + "inputs_embeds": 0,

68 + # the same shape as input_embeds

69 + "deepstack_input_embeds": 0

70 + })

71 +class Qwen3MoeLLMModel(Qwen3MoeModel):

72 +

73 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):

74 + super().__init__(vllm_config=vllm_config, prefix=prefix)

75 + if not get_pp_group().is_first_rank:

76 + assert self.start_layer >= len(

77 + vllm_config.model_config.hf_config.vision_config.

78 + deepstack_visual_indexes), (

79 + "start_layer should be greater than or equal to "

80 + "len(deepstack_visual_indexes)")

81 +

82 + def forward(

83 + self,

84 + input_ids: torch.Tensor,

85 + positions: torch.Tensor,

86 + intermediate_tensors: Optional[IntermediateTensors] = None,

87 + inputs_embeds: Optional[torch.Tensor] = None,

88 + deepstack_input_embeds: Optional[IntermediateTensors] = None,

89 + ) -> Union[torch.Tensor, IntermediateTensors]:

90 + if get_pp_group().is_first_rank:

91 + if inputs_embeds is not None:

92 + hidden_states = inputs_embeds

93 + else:

94 + hidden_states = self.get_input_embeddings(input_ids)

95 + residual = None

96 + else:

97 + assert intermediate_tensors is not None

98 + hidden_states = intermediate_tensors["hidden_states"]

99 + residual = intermediate_tensors["residual"]

100 + for layer_idx, layer in enumerate(

101 + self.layers[self.start_layer:self.end_layer]):

102 + layer_idx = layer_idx + self.start_layer

103 +

104 + hidden_states, residual = layer(

105 + positions,

106 + hidden_states,

107 + residual,

108 + )

109 +

110 + if deepstack_input_embeds is not None and \

111 + layer_idx in range(0, len(deepstack_input_embeds)):

112 + hidden_states = hidden_states + deepstack_input_embeds[

113 + f"deepstack_input_embeds_{layer_idx}"]

114 +

115 + if not get_pp_group().is_last_rank:

116 + return IntermediateTensors({

117 + "hidden_states": hidden_states,

118 + "residual": residual

119 + })

120 + hidden_states, _ = self.norm(hidden_states, residual)

121 + return hidden_states

122 +

123 + def load_fused_expert_weights(self, name: str, params_dict: dict,

124 + loaded_weight: torch.Tensor, shard_id: str,

125 + num_experts: int):

126 + param = params_dict[name]

127 + weight_loader = typing.cast(Callable[..., bool], param.weight_loader)

128 + for expert_id in range(num_experts):

129 + curr_expert_weight = loaded_weight[expert_id]

130 + success = weight_loader(param,

131 + curr_expert_weight,

132 + name,

133 + shard_id,

134 + expert_id,

135 + return_success=True)

136 + if not success:

137 + return False

138 + return True

139 +

140 + def load_weights(self, weights: Iterable[tuple[str,

141 + torch.Tensor]]) -> set[str]:

142 + stacked_params_mapping = [

143 + # (param_name, shard_name, shard_id)

144 + ("qkv_proj", "q_proj", "q"),

145 + ("qkv_proj", "k_proj", "k"),

146 + ("qkv_proj", "v_proj", "v"),

147 + ("gate_up_proj", "gate_proj", 0),

148 + ("gate_up_proj", "up_proj", 1),

149 + ]

150 + # Skip loading extra parameters for GPTQ/modelopt models.

151 + ignore_suffixes = (".bias", "_bias", ".k_scale", "_k_scale",

152 + ".v_scale", "_v_scale", ".weight_scale",

153 + "_weight_scale", ".input_scale", "_input_scale")

154 + params_dict = dict(self.named_parameters())

155 + loaded_params: set[str] = set()

156 + expert_params_mapping = self.get_expert_mapping()

157 + is_fused_expert = False

158 + fused_expert_params_mapping = [

159 + ("experts.w13_weight", "experts.gate_up_proj", 0, "w1"),

160 + ("experts.w2_weight", "experts.down_proj", 0, "w2"),

161 + ]

162 + num_experts = self.config.num_experts

163 + for name, loaded_weight in weights:

164 + for (param_name, weight_name, shard_id) in stacked_params_mapping:

165 + if ("experts.gate_up_proj" in name

166 + or "experts.down_proj" in name):

167 + is_fused_expert = True

168 + expert_params_mapping = fused_expert_params_mapping

169 +

170 + # Skip non-stacked layers and experts (experts handled below).

171 + if weight_name not in name:

172 + continue

173 + # We have mlp.experts[0].gate_proj in the checkpoint.

174 + # Since we handle the experts below in expert_params_mapping,

175 + # we need to skip here BEFORE we update the name, otherwise

176 + # name will be updated to mlp.experts[0].gate_up_proj, which

177 + # will then be updated below in expert_params_mapping

178 + # for mlp.experts[0].gate_gate_up_proj, which breaks load.

179 + if "mlp.experts" in name:

180 + continue

181 + name = name.replace(weight_name, param_name)

182 + # Skip loading extra parameters for GPTQ/modelopt models.

183 + if name.endswith(ignore_suffixes) and name not in params_dict:

184 + continue

185 + # Skip layers on other devices.

186 + if is_pp_missing_parameter(name, self):

187 + continue

188 + if name.endswith("scale"):

189 + # Remapping the name of FP8 kv-scale.

190 + name = maybe_remap_kv_scale_name(name, params_dict)

191 + if name is None:

192 + continue

193 + if name not in params_dict:

194 + continue

195 + param = params_dict[name]

196 + weight_loader = getattr(param, "weight_loader",

197 + default_weight_loader)

198 + if weight_loader == default_weight_loader:

199 + weight_loader(param, loaded_weight)

200 + else:

201 + weight_loader(param, loaded_weight, shard_id)

202 + break

203 + else:

204 + is_expert_weight = False

205 + for mapping in expert_params_mapping:

206 + param_name, weight_name, expert_id, shard_id = mapping

207 + if weight_name not in name:

208 + continue

209 + # Anyway, this is an expert weight and should not be

210 + # attempted to load as other weights later

211 + is_expert_weight = True

212 + name_mapped = name.replace(weight_name, param_name)

213 + if is_fused_expert:

214 + loaded_weight = loaded_weight.transpose(-1,

215 + -2) # no bias

216 + if "experts.gate_up_proj" in name:

217 + loaded_weight = loaded_weight.chunk(2, dim=-2)

218 + success_w1 = self.load_fused_expert_weights(

219 + name_mapped, params_dict, loaded_weight[0],

220 + "w1", num_experts)

221 + success_w3 = self.load_fused_expert_weights(

222 + name_mapped, params_dict, loaded_weight[1],

223 + "w3", num_experts)

224 + success = success_w1 and success_w3

225 + else:

226 + # down_proj

227 + success = self.load_fused_expert_weights(

228 + name_mapped, params_dict, loaded_weight,

229 + shard_id, num_experts)

230 + else:

231 + if is_pp_missing_parameter(name_mapped, self):

232 + continue

233 + # Skip loading extra parameters for GPTQ/modelopt models

234 + if name_mapped.endswith(

235 + ignore_suffixes

236 + ) and name_mapped not in params_dict:

237 + continue

238 + param = params_dict[name_mapped]

239 + # We should ask the weight loader to return success or

240 + # not here since otherwise we may skip experts with

241 + # other available replicas.

242 + weight_loader = typing.cast(Callable[..., bool],

243 + param.weight_loader)

244 + success = weight_loader(param,

245 + loaded_weight,

246 + name_mapped,

247 + shard_id=shard_id,

248 + expert_id=expert_id,

249 + return_success=True)

250 + if success:

251 + name = name_mapped

252 + break

253 + else:

254 + if is_expert_weight:

255 + # We've checked that this is an expert weight

256 + # However it's not mapped locally to this rank

257 + # So we simply skip it

258 + continue

259 + # Skip loading extra parameters for GPTQ/modelopt models.

260 + if name.endswith(

261 + ignore_suffixes) and name not in params_dict:

262 + continue

263 + # Skip layers on other devices.

264 + if is_pp_missing_parameter(name, self):

265 + continue

266 + # Remapping the name of FP8 kv-scale.

267 + if name.endswith("kv_scale"):

268 + remapped_kv_scale_name = name.replace(

269 + ".kv_scale", ".attn.kv_scale")

270 + if remapped_kv_scale_name not in params_dict:

271 + logger.warning_once(

272 + "Found kv scale in the checkpoint (e.g. %s), but not found the expected name in the model (e.g. %s). kv-scale is not loaded.", # noqa: E501

273 + name,

274 + remapped_kv_scale_name,

275 + )

276 + continue

277 + else:

278 + name = remapped_kv_scale_name

279 + param = params_dict[name]

280 + weight_loader = getattr(param, "weight_loader",

281 + default_weight_loader)

282 + weight_loader(param, loaded_weight)

283 + loaded_params.add(name)

284 + return loaded_params

285 +

286 +

287 +class Qwen3MoeLLMForCausalLM(Qwen3MoeForCausalLM):

288 +

289 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):

290 + super(Qwen3MoeForCausalLM, self).__init__()

291 + self.config = vllm_config.model_config.hf_config.text_config

292 + self.quant_config = vllm_config.quant_config

293 + self.model = Qwen3MoeLLMModel(vllm_config=vllm_config,

294 + prefix=maybe_prefix(prefix, "model"))

295 + self.lm_head = ParallelLMHead(self.config.vocab_size,

296 + self.config.hidden_size,

297 + quant_config=self.quant_config)

298 + if self.config.tie_word_embeddings:

299 + self.lm_head.weight = self.model.embed_tokens.weight

300 + self.logits_processor = LogitsProcessor(self.config.vocab_size)

301 + self.make_empty_intermediate_tensors = (

302 + self.model.make_empty_intermediate_tensors)

303 +

304 +

305 +@MULTIMODAL_REGISTRY.register_processor(Qwen3VLMultiModalProcessor,

306 + info=Qwen3VLMoeProcessingInfo,

307 + dummy_inputs=Qwen3VLDummyInputsBuilder)

308 +class Qwen3VLMoeForConditionalGeneration(Qwen3VLForConditionalGeneration):

309 +

310 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):

311 + super(Qwen3VLForConditionalGeneration, self).__init__()

312 + config: Qwen3VLMoeConfig = vllm_config.model_config.hf_config

313 + quant_config = vllm_config.quant_config

314 + multimodal_config = vllm_config.model_config.multimodal_config

315 +

316 + self.config = config

317 + self.multimodal_config = multimodal_config

318 +

319 + self.visual = Qwen3_VisionTransformer(

320 + config.vision_config,

321 + norm_eps=getattr(config, "rms_norm_eps", 1e-6),

322 + quant_config=self._maybe_ignore_quant_config(quant_config),

323 + prefix=maybe_prefix(prefix, "visual"),

324 + )

325 +

326 + self.language_model = Qwen3MoeLLMForCausalLM(vllm_config=vllm_config,

327 + prefix=maybe_prefix(

328 + prefix,

329 + "language_model"))

330 +

331 + self.make_empty_intermediate_tensors = (

332 + self.language_model.make_empty_intermediate_tensors)

333 +

334 + self.use_deepstack = hasattr(config.vision_config,

335 + 'deepstack_visual_indexes')

336 + self.deepstack_num_level = len(

337 + config.vision_config.deepstack_visual_indexes

338 + ) if self.use_deepstack else 0

339 + # register buffer for deepstack

340 + self.deepstack_input_embeds = [

341 + torch.zeros(vllm_config.scheduler_config.max_num_batched_tokens,

342 + config.text_config.hidden_size)

343 + for _ in range(self.deepstack_num_level)

344 + ] if self.use_deepstack else None

vllm/model_executor/models/registry.py

+3 -1

@@ -259,11 +259,13 @@ _MULTIMODAL_MODELS = {

259 "Qwen2AudioForConditionalGeneration": ("qwen2_audio", "Qwen2AudioForConditionalGeneration"), # noqa: E501

260 "Qwen2_5OmniModel": ("qwen2_5_omni_thinker", "Qwen2_5OmniThinkerForConditionalGeneration"), # noqa: E501

261 "Qwen2_5OmniForConditionalGeneration": ("qwen2_5_omni_thinker", "Qwen2_5OmniThinkerForConditionalGeneration"), # noqa: E501

262 - "UltravoxModel": ("ultravox", "UltravoxModel"),

262 + "Qwen3VLForConditionalGeneration": ("qwen3_vl", "Qwen3VLForConditionalGeneration"), # noqa: E501

263 + "Qwen3VLMoeForConditionalGeneration": ("qwen3_vl_moe", "Qwen3VLMoeForConditionalGeneration"), # noqa: E501

264 "SkyworkR1VChatModel": ("skyworkr1v", "SkyworkR1VChatModel"),

265 "Step3VLForConditionalGeneration": ("step3_vl", "Step3VLForConditionalGeneration"), # noqa: E501

266 "TarsierForConditionalGeneration": ("tarsier", "TarsierForConditionalGeneration"), # noqa: E501

267 "Tarsier2ForConditionalGeneration": ("qwen2_vl", "Tarsier2ForConditionalGeneration"), # noqa: E501

268 + "UltravoxModel": ("ultravox", "UltravoxModel"),

269 "VoxtralForConditionalGeneration": ("voxtral", "VoxtralForConditionalGeneration"), # noqa: E501

270 # [Encoder-decoder]

271 "WhisperForConditionalGeneration": ("whisper", "WhisperForConditionalGeneration"), # noqa: E501

vllm/multimodal/video.py

+1 -1

@@ -156,7 +156,7 @@ class OpenCVVideoBackend(VideoLoader):

156 # can cause incorrect timestamp calculation without num_frames=-1.

157 metadata = {

158 "total_num_frames": num_frames,

159 - "fps": original_fps,

159 + "fps": num_frames / duration,

160 "duration": duration,

161 "video_backend": "opencv",

162 "frames_indices": list(range(num_frames)),