A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[Bugfix] Support 2D input shape in MoE layer (#6287)
Woosuk Kwon committed
Jul 10, 2024 at 06:03 UTC
e72ae80b06405ea92b703c8979f046d68e970c94
2 files changed
+7
-4
vllm/model_executor/models/mixtral.py
+3
-2
@@ -88,12 +88,13 @@ class MixtralMoE(nn.Module):
88
tp_size=tp_size)
89
90
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
91
- num_tokens, hidden_size = hidden_states.shape
91
+ # NOTE: hidden_states can have either 1D or 2D shape.
92
+ orig_shape = hidden_states.shape
93
hidden_states = hidden_states.view(-1, self.hidden_size)
94
# router_logits: (num_tokens, n_experts)
95
router_logits, _ = self.gate(hidden_states)
96
final_hidden_states = self.experts(hidden_states, router_logits)
96
- return final_hidden_states.view(num_tokens, hidden_size)
97
+ return final_hidden_states.view(orig_shape)
98
99
100
class MixtralAttention(nn.Module):
vllm/model_executor/models/qwen2_moe.py
+4
-2
@@ -126,7 +126,9 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
126
bias=False)
127
128
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
129
- num_tokens, hidden_dim = hidden_states.shape
129
+ # NOTE: hidden_states can have either 1D or 2D shape.
130
+ orig_shape = hidden_states.shape
131
+ hidden_dim = hidden_states.shape[-1]
132
hidden_states = hidden_states.view(-1, hidden_dim)
133
shared_output = None
134
if self.shared_expert is not None:
@@ -145,7 +147,7 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
147
final_hidden_states = tensor_model_parallel_all_reduce(
148
final_hidden_states)
149
148
- return final_hidden_states.view(num_tokens, hidden_dim)
150
+ return final_hidden_states.view(orig_shape)
151
152
153
class Qwen2MoeAttention(nn.Module):