A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[MM][Bugfix] Replace `PatchEmbed`'s conv3d to linear layer (#27418)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Roger Wang <hey@rogerw.io>
Isotr0py committed
Oct 24, 2025 at 15:32 UTC
42efe609ba75eb1b0bc06ae635778b2bc0aa4e7a
6 files changed
+97
-42
vllm/model_executor/models/glm4_1v.py
+13
-8
@@ -60,6 +60,7 @@ from vllm.model_executor.layers.linear import (
60
ColumnParallelLinear,
61
MergedColumnParallelLinear,
62
QKVParallelLinear,
63
+ ReplicatedLinear,
64
RowParallelLinear,
65
)
66
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -98,7 +99,11 @@ from .utils import (
99
init_vllm_registered_model,
100
maybe_prefix,
101
)
101
-from .vision import get_vit_attn_backend, run_dp_sharded_mrope_vision_model
102
+from .vision import (
103
+ conv3d_to_linear_weight,
104
+ get_vit_attn_backend,
105
+ run_dp_sharded_mrope_vision_model,
106
+)
107
108
logger = init_logger(__name__)
109
@@ -478,18 +483,15 @@ class Glm4vVisionPatchEmbed(nn.Module):
483
self.hidden_size = hidden_size
484
485
kernel_size = (temporal_patch_size, patch_size, patch_size)
481
- self.proj = nn.Conv3d(
482
- in_channels,
486
+ self.proj = ReplicatedLinear(
487
+ in_channels * math.prod(kernel_size),
488
hidden_size,
484
- kernel_size=kernel_size,
485
- stride=kernel_size,
489
bias=True,
490
+ return_bias=False,
491
)
492
493
def forward(self, x: torch.Tensor) -> torch.Tensor:
490
- L, C = x.shape
491
- x = x.view(L, -1, self.temporal_patch_size, self.patch_size, self.patch_size)
492
- x = self.proj(x).view(L, self.hidden_size)
494
+ x = self.proj(x)
495
return x
496
497
@@ -887,6 +889,9 @@ class Glm4vVisionTransformer(nn.Module):
889
loaded_params: set[str] = set()
890
891
for name, loaded_weight in weights:
892
+ if name.endswith("patch_embed.proj.weight"):
893
+ loaded_weight = conv3d_to_linear_weight(loaded_weight)
894
+
895
for param_name, weight_name, shard_id in stacked_params_mapping:
896
if weight_name not in name:
897
continue
vllm/model_executor/models/qwen2_5_vl.py
+14
-8
@@ -26,6 +26,7 @@
26
# limitations under the License.
27
"""Inference-only Qwen2.5-VL model compatible with HuggingFace weights."""
28
29
+import math
30
from collections.abc import Callable, Iterable, Mapping, Sequence
31
from functools import lru_cache, partial
32
from typing import Annotated, Any, Literal, TypeAlias
@@ -56,6 +57,7 @@ from vllm.model_executor.layers.linear import (
57
ColumnParallelLinear,
58
MergedColumnParallelLinear,
59
QKVParallelLinear,
60
+ ReplicatedLinear,
61
RowParallelLinear,
62
)
63
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -98,7 +100,11 @@ from .utils import (
100
init_vllm_registered_model,
101
maybe_prefix,
102
)
101
-from .vision import get_vit_attn_backend, run_dp_sharded_mrope_vision_model
103
+from .vision import (
104
+ conv3d_to_linear_weight,
105
+ get_vit_attn_backend,
106
+ run_dp_sharded_mrope_vision_model,
107
+)
108
109
logger = init_logger(__name__)
110
@@ -532,18 +538,15 @@ class Qwen2_5_VisionPatchEmbed(nn.Module):
538
self.hidden_size = hidden_size
539
540
kernel_size = (temporal_patch_size, patch_size, patch_size)
535
- self.proj = nn.Conv3d(
536
- in_channels,
541
+ self.proj = ReplicatedLinear(
542
+ in_channels * math.prod(kernel_size),
543
hidden_size,
538
- kernel_size=kernel_size,
539
- stride=kernel_size,
544
bias=False,
545
+ return_bias=False,
546
)
547
548
def forward(self, x: torch.Tensor) -> torch.Tensor:
544
- L, C = x.shape
545
- x = x.view(L, -1, self.temporal_patch_size, self.patch_size, self.patch_size)
546
- x = self.proj(x).view(L, self.hidden_size)
549
+ x = self.proj(x)
550
return x
551
552
@@ -950,6 +953,9 @@ class Qwen2_5_VisionTransformer(nn.Module):
953
loaded_params: set[str] = set()
954
955
for name, loaded_weight in weights:
956
+ if name.endswith("patch_embed.proj.weight"):
957
+ loaded_weight = conv3d_to_linear_weight(loaded_weight)
958
+
959
for param_name, weight_name, shard_id in stacked_params_mapping:
960
if weight_name not in name:
961
continue
vllm/model_executor/models/qwen2_vl.py
+18
-9
@@ -25,6 +25,7 @@
25
# limitations under the License.
26
"""Inference-only Qwen2-VL model compatible with HuggingFace weights."""
27
28
+import math
29
from collections.abc import Callable, Iterable, Mapping, Sequence
30
from functools import partial
31
from typing import Annotated, Any, Literal, TypeAlias
@@ -53,7 +54,11 @@ from vllm.distributed import parallel_state, tensor_model_parallel_all_gather
54
from vllm.distributed import utils as dist_utils
55
from vllm.logger import init_logger
56
from vllm.model_executor.layers.activation import QuickGELU
56
-from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
57
+from vllm.model_executor.layers.linear import (
58
+ ColumnParallelLinear,
59
+ ReplicatedLinear,
60
+ RowParallelLinear,
61
+)
62
from vllm.model_executor.layers.quantization import QuantizationConfig
63
from vllm.model_executor.layers.rotary_embedding.common import (
64
dispatch_rotary_emb_function,
@@ -100,7 +105,11 @@ from .utils import (
105
init_vllm_registered_model,
106
maybe_prefix,
107
)
103
-from .vision import get_vit_attn_backend, run_dp_sharded_mrope_vision_model
108
+from .vision import (
109
+ conv3d_to_linear_weight,
110
+ get_vit_attn_backend,
111
+ run_dp_sharded_mrope_vision_model,
112
+)
113
114
logger = init_logger(__name__)
115
@@ -561,18 +570,15 @@ class Qwen2VisionPatchEmbed(nn.Module):
570
self.embed_dim = embed_dim
571
572
kernel_size = (temporal_patch_size, patch_size, patch_size)
564
- self.proj = nn.Conv3d(
565
- in_channels,
573
+ self.proj = ReplicatedLinear(
574
+ in_channels * math.prod(kernel_size),
575
embed_dim,
567
- kernel_size=kernel_size,
568
- stride=kernel_size,
576
bias=False,
577
+ return_bias=False,
578
)
579
580
def forward(self, x: torch.Tensor) -> torch.Tensor:
573
- L, C = x.shape
574
- x = x.view(L, -1, self.temporal_patch_size, self.patch_size, self.patch_size)
575
- x = self.proj(x).view(L, self.embed_dim)
581
+ x = self.proj(x)
582
return x
583
584
@@ -835,6 +841,9 @@ class Qwen2VisionTransformer(nn.Module):
841
loaded_params: set[str] = set()
842
843
for name, loaded_weight in weights:
844
+ if name.endswith("patch_embed.proj.weight"):
845
+ loaded_weight = conv3d_to_linear_weight(loaded_weight)
846
+
847
for param_name, weight_name, shard_id in stacked_params_mapping:
848
if weight_name not in name:
849
continue
vllm/model_executor/models/qwen3_omni_moe_thinker.py
+18
-8
@@ -22,6 +22,7 @@
22
# limitations under the License.
23
"""Inference-only Qwen3-Omni-Moe model (thinker part)."""
24
25
+import math
26
from collections.abc import Callable, Iterable, Mapping, Sequence
27
from functools import partial
28
from typing import Any
@@ -53,7 +54,11 @@ from vllm.config import VllmConfig
54
from vllm.distributed import get_pp_group
55
from vllm.logger import init_logger
56
from vllm.model_executor.layers.activation import _ACTIVATION_REGISTRY
56
-from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
57
+from vllm.model_executor.layers.linear import (
58
+ ColumnParallelLinear,
59
+ ReplicatedLinear,
60
+ RowParallelLinear,
61
+)
62
from vllm.model_executor.layers.logits_processor import LogitsProcessor
63
from vllm.model_executor.layers.quantization import QuantizationConfig
64
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
@@ -98,7 +103,11 @@ from .utils import (
103
_merge_multimodal_embeddings,
104
maybe_prefix,
105
)
101
-from .vision import get_llm_pos_ids_for_vision, get_vit_attn_backend
106
+from .vision import (
107
+ conv3d_to_linear_weight,
108
+ get_llm_pos_ids_for_vision,
109
+ get_vit_attn_backend,
110
+)
111
112
try:
113
import flash_attn
@@ -131,18 +140,16 @@ class Qwen3_VisionPatchEmbed(nn.Module):
140
self.hidden_size = hidden_size
141
142
kernel_size = (temporal_patch_size, patch_size, patch_size)
134
- self.proj = nn.Conv3d(
135
- in_channels,
143
+ self.proj = ReplicatedLinear(
144
+ in_channels * math.prod(kernel_size),
145
hidden_size,
137
- kernel_size=kernel_size,
138
- stride=kernel_size,
146
bias=True,
147
+ return_bias=False,
148
)
149
150
def forward(self, x: torch.Tensor) -> torch.Tensor:
151
L, C = x.shape
144
- x = x.view(L, -1, self.temporal_patch_size, self.patch_size, self.patch_size)
145
- x = self.proj(x).view(L, self.hidden_size)
152
+ x = self.proj(x)
153
return x
154
155
@@ -559,6 +566,9 @@ class Qwen3Omni_VisionTransformer(nn.Module):
566
loaded_params: set[str] = set()
567
568
for name, loaded_weight in weights:
569
+ if name.endswith("patch_embed.proj.weight"):
570
+ loaded_weight = conv3d_to_linear_weight(loaded_weight)
571
+
572
for param_name, weight_name, shard_id in stacked_params_mapping:
573
if weight_name not in name:
574
continue
vllm/model_executor/models/qwen3_vl.py
+18
-9
@@ -24,6 +24,7 @@
24
# limitations under the License.
25
"""Inference-only Qwen3VL model compatible with HuggingFace weights."""
26
27
+import math
28
from collections.abc import Callable, Iterable, Mapping, Sequence
29
from functools import partial
30
from itertools import islice
@@ -56,7 +57,11 @@ from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions
57
from vllm.distributed import get_pp_group
58
from vllm.logger import init_logger
59
from vllm.model_executor.layers.activation import _ACTIVATION_REGISTRY
59
-from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
60
+from vllm.model_executor.layers.linear import (
61
+ ColumnParallelLinear,
62
+ ReplicatedLinear,
63
+ RowParallelLinear,
64
+)
65
from vllm.model_executor.layers.logits_processor import LogitsProcessor
66
from vllm.model_executor.layers.quantization import QuantizationConfig
67
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
@@ -107,7 +112,11 @@ from .utils import (
112
_merge_multimodal_embeddings,
113
maybe_prefix,
114
)
110
-from .vision import get_vit_attn_backend, run_dp_sharded_mrope_vision_model
115
+from .vision import (
116
+ conv3d_to_linear_weight,
117
+ get_vit_attn_backend,
118
+ run_dp_sharded_mrope_vision_model,
119
+)
120
121
logger = init_logger(__name__)
122
@@ -129,18 +138,15 @@ class Qwen3_VisionPatchEmbed(nn.Module):
138
self.hidden_size = hidden_size
139
140
kernel_size = (temporal_patch_size, patch_size, patch_size)
132
- self.proj = nn.Conv3d(
133
- in_channels,
141
+ self.proj = ReplicatedLinear(
142
+ in_channels * math.prod(kernel_size),
143
hidden_size,
135
- kernel_size=kernel_size,
136
- stride=kernel_size,
144
bias=True,
145
+ return_bias=False,
146
)
147
148
def forward(self, x: torch.Tensor) -> torch.Tensor:
141
- L, C = x.shape
142
- x = x.view(L, -1, self.temporal_patch_size, self.patch_size, self.patch_size)
143
- x = self.proj(x).view(L, self.hidden_size)
149
+ x = self.proj(x)
150
return x
151
152
@@ -576,6 +582,9 @@ class Qwen3_VisionTransformer(nn.Module):
582
loaded_params: set[str] = set()
583
584
for name, loaded_weight in weights:
585
+ if name.endswith("patch_embed.proj.weight"):
586
+ loaded_weight = conv3d_to_linear_weight(loaded_weight)
587
+
588
for param_name, weight_name, shard_id in stacked_params_mapping:
589
if weight_name not in name:
590
continue
vllm/model_executor/models/vision.py
+16
@@ -544,3 +544,19 @@ def get_llm_pos_ids_for_vision(
544
llm_pos_ids_list.append(_llm_pos_ids + start_idx)
545
llm_pos_ids = torch.cat(llm_pos_ids_list, dim=1)
546
return llm_pos_ids
547
+
548
+
549
+# Due to a performance regression with Conv3D in PyTorch2.9, we reshape
550
+# Conv3D weights to Linear weights for better performance.
551
+# See: https://github.com/vllm-project/vllm/issues/27406
552
+# and https://github.com/pytorch/pytorch/issues/166122
553
+# FIXME(Isotr0py): Revert the PR introduces this workaround
554
+# (https://github.com/vllm-project/vllm/pull/27418),
555
+# once the performance issue is resolved in PyTorch.
556
+def conv3d_to_linear_weight(conv3d_weight: torch.Tensor) -> torch.Tensor:
557
+ """
558
+ Reshape Conv3D weight to Linear weight. Only work when kernel_size==stride.
559
+ """
560
+ out_channels, in_channels, kt, kh, kw = conv3d_weight.shape
561
+ linear_weight = conv3d_weight.reshape(out_channels, in_channels * kt * kh * kw)
562
+ return linear_weight