A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 2, 17:01 UTC
Fix Qwen3-VL and Qwen3-omni-thinker accuracy degradation from deepstack inputs under torch.compile (#43617)
Signed-off-by: Dakai An <dakaian108@gmail.com>
Dakai An committed
May 28, 2026 at 06:34 UTC
5963c194787d30ed4a49c1e2e01010d8dffe1e79
2 files changed
+28
-22
vllm/model_executor/models/qwen3_omni_moe_thinker.py
+14
-11
@@ -1778,8 +1778,8 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
1778
) -> IntermediateTensors | None:
1779
if not getattr(self, "deepstack_input_embeds", None):
1780
return None # If vision tower is skipped
1781
- if getattr(self, "deepstack_input_embeds_num_tokens", 0) == 0:
1782
- return None
1781
+ if num_tokens > self.deepstack_input_embeds[0].size(0):
1782
+ self._resize_deepstack_input_embeds(num_tokens)
1783
1784
# get deepstack_input_embeds from buffer, and clear the buffer
1785
return IntermediateTensors(
@@ -1791,6 +1791,17 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
1791
}
1792
)
1793
1794
+ def _resize_deepstack_input_embeds(self, num_tokens: int) -> None:
1795
+ self.deepstack_input_embeds = [
1796
+ torch.zeros(
1797
+ num_tokens,
1798
+ self.config.text_config.hidden_size,
1799
+ device=self.deepstack_input_embeds[0].device,
1800
+ dtype=self.deepstack_input_embeds[0].dtype,
1801
+ )
1802
+ for _ in range(self.deepstack_num_level)
1803
+ ]
1804
+
1805
def _set_deepstack_input_embeds(self, deepstack_input_embeds: torch.Tensor) -> None:
1806
if not getattr(self, "deepstack_input_embeds", None):
1807
return
@@ -1798,15 +1809,7 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
1809
# set deepstack_input_embeds to buffer
1810
num_tokens = deepstack_input_embeds.size(1)
1811
if num_tokens > self.deepstack_input_embeds[0].size(0):
1801
- self.deepstack_input_embeds = [
1802
- torch.zeros(
1803
- num_tokens,
1804
- self.config.text_config.hidden_size,
1805
- device=self.deepstack_input_embeds[0].device,
1806
- dtype=self.deepstack_input_embeds[0].dtype,
1807
- )
1808
- for _ in range(self.deepstack_num_level)
1809
- ]
1812
+ self._resize_deepstack_input_embeds(num_tokens)
1813
for idx in range(self.deepstack_num_level):
1814
self.deepstack_input_embeds[idx][:num_tokens].copy_(
1815
deepstack_input_embeds[idx]
vllm/model_executor/models/qwen3_vl.py
+14
-11
@@ -1715,8 +1715,8 @@ class Qwen3VLForConditionalGeneration(
1715
) -> IntermediateTensors | None:
1716
if not getattr(self, "deepstack_input_embeds", None):
1717
return None # If vision tower is skipped
1718
- if getattr(self, "deepstack_input_embeds_num_tokens", 0) == 0:
1719
- return None
1718
+ if num_tokens > self.deepstack_input_embeds[0].size(0):
1719
+ self._resize_deepstack_input_embeds(num_tokens)
1720
1721
# get deepstack_input_embeds from buffer, and clear the buffer
1722
return IntermediateTensors(
@@ -1728,6 +1728,17 @@ class Qwen3VLForConditionalGeneration(
1728
}
1729
)
1730
1731
+ def _resize_deepstack_input_embeds(self, num_tokens: int) -> None:
1732
+ self.deepstack_input_embeds = [
1733
+ torch.zeros(
1734
+ num_tokens,
1735
+ self.config.text_config.hidden_size,
1736
+ device=self.deepstack_input_embeds[0].device,
1737
+ dtype=self.deepstack_input_embeds[0].dtype,
1738
+ )
1739
+ for _ in range(self.deepstack_num_level)
1740
+ ]
1741
+
1742
def _set_deepstack_input_embeds(self, deepstack_input_embeds: torch.Tensor) -> None:
1743
if not getattr(self, "deepstack_input_embeds", None):
1744
return
@@ -1735,15 +1746,7 @@ class Qwen3VLForConditionalGeneration(
1746
# set deepstack_input_embeds to buffer
1747
num_tokens = deepstack_input_embeds.size(1)
1748
if num_tokens > self.deepstack_input_embeds[0].size(0):
1738
- self.deepstack_input_embeds = [
1739
- torch.zeros(
1740
- num_tokens,
1741
- self.config.text_config.hidden_size,
1742
- device=self.deepstack_input_embeds[0].device,
1743
- dtype=self.deepstack_input_embeds[0].dtype,
1744
- )
1745
- for _ in range(self.deepstack_num_level)
1746
- ]
1749
+ self._resize_deepstack_input_embeds(num_tokens)
1750
for idx in range(self.deepstack_num_level):
1751
self.deepstack_input_embeds[idx][:num_tokens].copy_(
1752
deepstack_input_embeds[idx]