[Bugfix] Skip Qwen3 deepstack buffers without vision (#49397)
Signed-off-by: WEI CHENG CHIU <waynehacking8@gmail.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
Wei-Cheng (Wayne) Chiu committed
Aug 5, 2026 at 12:46 UTC
7794b1e08bf505ff28664515ffaaeeec955ab796
3 files changed
+64
-61
vllm/model_executor/models/qwen3_omni_moe_thinker.py
+21
-21
@@ -102,6 +102,7 @@ from .qwen2_5_vl import (
102
from .qwen3_moe import Qwen3MoeForCausalLM, Qwen3MoeModel
103
from .utils import (
104
AutoWeightsLoader,
105
+ StageMissingLayer,
106
WeightsMapper,
107
_merge_multimodal_embeddings,
108
maybe_prefix,
@@ -1620,9 +1621,17 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
1621
prefix=maybe_prefix(prefix, "audio_tower"),
1622
)
1623
1624
+ with self._mark_tower_model(vllm_config, {"image", "video"}):
1625
+ self.visual = Qwen3Omni_VisionTransformer(
1626
+ vision_config=thinker_config.vision_config,
1627
+ norm_eps=getattr(thinker_config.text_config, "rms_norm_eps", 1e-6),
1628
+ quant_config=quant_config,
1629
+ prefix=maybe_prefix(prefix, "visual"),
1630
+ )
1631
+
1632
self.use_deepstack = hasattr(
1633
thinker_config.vision_config, "deepstack_visual_indexes"
1625
- )
1634
+ ) and not isinstance(self.visual, StageMissingLayer)
1635
self.deepstack_num_level = (
1636
len(thinker_config.vision_config.deepstack_visual_indexes)
1637
if self.use_deepstack
@@ -1631,26 +1640,17 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
1640
self.visual_dim = thinker_config.vision_config.out_hidden_size
1641
self.multiscale_dim = self.visual_dim * self.deepstack_num_level
1642
1634
- with self._mark_tower_model(vllm_config, {"image", "video"}):
1635
- self.visual = Qwen3Omni_VisionTransformer(
1636
- vision_config=thinker_config.vision_config,
1637
- norm_eps=getattr(thinker_config.text_config, "rms_norm_eps", 1e-6),
1638
- quant_config=quant_config,
1639
- prefix=maybe_prefix(prefix, "visual"),
1640
- )
1641
-
1642
- # register buffer for deepstack
1643
- if self.use_deepstack:
1644
- self.deepstack_input_embeds = [
1645
- torch.zeros(
1646
- vllm_config.scheduler_config.max_num_batched_tokens,
1647
- thinker_config.text_config.hidden_size,
1648
- )
1649
- for _ in range(self.deepstack_num_level)
1650
- ]
1651
- # Tracks the valid token span currently stored in the buffer.
1652
- # Zero means there is no active deepstack payload to consume.
1653
- self.deepstack_input_embeds_num_tokens = 0
1643
+ if self.use_deepstack:
1644
+ self.deepstack_input_embeds = [
1645
+ torch.zeros(
1646
+ vllm_config.scheduler_config.max_num_batched_tokens,
1647
+ thinker_config.text_config.hidden_size,
1648
+ )
1649
+ for _ in range(self.deepstack_num_level)
1650
+ ]
1651
+ # Tracks the valid token span currently stored in the buffer.
1652
+ # Zero means there is no active deepstack payload to consume.
1653
+ self.deepstack_input_embeds_num_tokens = 0
1654
1655
with self._mark_language_model(vllm_config):
1656
self.language_model = Qwen3MoeLLMForCausalLM(
vllm/model_executor/models/qwen3_vl.py
+23
-21
@@ -142,6 +142,7 @@ from .qwen3 import Qwen3ForCausalLM, Qwen3Model
142
from .utils import (
143
AutoWeightsLoader,
144
PPMissingLayer,
145
+ StageMissingLayer,
146
WeightsMapper,
147
_merge_multimodal_embeddings,
148
maybe_prefix,
@@ -1769,15 +1770,6 @@ class Qwen3VLForConditionalGeneration(
1770
self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
1771
self._init_video_pruning(multimodal_config)
1772
1772
- self.use_deepstack = hasattr(config.vision_config, "deepstack_visual_indexes")
1773
- self.deepstack_num_level = (
1774
- len(config.vision_config.deepstack_visual_indexes)
1775
- if self.use_deepstack
1776
- else 0
1777
- )
1778
- self.visual_dim = config.vision_config.out_hidden_size
1779
- self.multiscale_dim = self.visual_dim * self.deepstack_num_level
1780
-
1773
with self._mark_tower_model(vllm_config, {"image", "video"}):
1774
self.visual = Qwen3_VisionTransformer(
1775
config.vision_config,
@@ -1786,18 +1778,28 @@ class Qwen3VLForConditionalGeneration(
1778
prefix=maybe_prefix(prefix, "visual"),
1779
)
1780
1789
- # register buffer for deepstack
1790
- if self.use_deepstack:
1791
- self.deepstack_input_embeds = [
1792
- torch.zeros(
1793
- vllm_config.scheduler_config.max_num_batched_tokens,
1794
- config.text_config.hidden_size,
1795
- )
1796
- for _ in range(self.deepstack_num_level)
1797
- ]
1798
- # Tracks the valid token span currently stored in the buffer.
1799
- # Zero means there is no active deepstack payload to consume.
1800
- self.deepstack_input_embeds_num_tokens = 0
1781
+ self.use_deepstack = hasattr(
1782
+ config.vision_config, "deepstack_visual_indexes"
1783
+ ) and not isinstance(self.visual, StageMissingLayer)
1784
+ self.deepstack_num_level = (
1785
+ len(config.vision_config.deepstack_visual_indexes)
1786
+ if self.use_deepstack
1787
+ else 0
1788
+ )
1789
+ self.visual_dim = config.vision_config.out_hidden_size
1790
+ self.multiscale_dim = self.visual_dim * self.deepstack_num_level
1791
+
1792
+ if self.use_deepstack:
1793
+ self.deepstack_input_embeds = [
1794
+ torch.zeros(
1795
+ vllm_config.scheduler_config.max_num_batched_tokens,
1796
+ config.text_config.hidden_size,
1797
+ )
1798
+ for _ in range(self.deepstack_num_level)
1799
+ ]
1800
+ # Tracks the valid token span currently stored in the buffer.
1801
+ # Zero means there is no active deepstack payload to consume.
1802
+ self.deepstack_input_embeds_num_tokens = 0
1803
1804
with self._mark_language_model(vllm_config):
1805
self.language_model = Qwen3LLMForCausalLM(
vllm/model_executor/models/qwen3_vl_moe.py
+20
-19
@@ -54,7 +54,7 @@ from .qwen3_vl import (
54
Qwen3VLMultiModalProcessor,
55
Qwen3VLProcessingInfo,
56
)
57
-from .utils import maybe_prefix
57
+from .utils import StageMissingLayer, maybe_prefix
58
59
logger = init_logger(__name__)
60
@@ -222,15 +222,6 @@ class Qwen3VLMoeForConditionalGeneration(
222
self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
223
self._init_video_pruning(multimodal_config)
224
225
- self.use_deepstack = hasattr(config.vision_config, "deepstack_visual_indexes")
226
- self.deepstack_num_level = (
227
- len(config.vision_config.deepstack_visual_indexes)
228
- if self.use_deepstack
229
- else 0
230
- )
231
- self.visual_dim = config.vision_config.out_hidden_size
232
- self.multiscale_dim = self.visual_dim * self.deepstack_num_level
233
-
225
with self._mark_tower_model(vllm_config, {"image", "video"}):
226
self.visual = Qwen3_VisionTransformer(
227
config.vision_config,
@@ -239,15 +230,25 @@ class Qwen3VLMoeForConditionalGeneration(
230
prefix=maybe_prefix(prefix, "visual"),
231
)
232
242
- # register buffer for deepstack
243
- if self.use_deepstack:
244
- self.deepstack_input_embeds = [
245
- torch.zeros(
246
- vllm_config.scheduler_config.max_num_batched_tokens,
247
- config.text_config.hidden_size,
248
- )
249
- for _ in range(self.deepstack_num_level)
250
- ]
233
+ self.use_deepstack = hasattr(
234
+ config.vision_config, "deepstack_visual_indexes"
235
+ ) and not isinstance(self.visual, StageMissingLayer)
236
+ self.deepstack_num_level = (
237
+ len(config.vision_config.deepstack_visual_indexes)
238
+ if self.use_deepstack
239
+ else 0
240
+ )
241
+ self.visual_dim = config.vision_config.out_hidden_size
242
+ self.multiscale_dim = self.visual_dim * self.deepstack_num_level
243
+
244
+ if self.use_deepstack:
245
+ self.deepstack_input_embeds = [
246
+ torch.zeros(
247
+ vllm_config.scheduler_config.max_num_batched_tokens,
248
+ config.text_config.hidden_size,
249
+ )
250
+ for _ in range(self.deepstack_num_level)
251
+ ]
252
253
with self._mark_language_model(vllm_config):
254
self.language_model = Qwen3MoeLLMForCausalLM(