A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
[BugFix] 1D query fix for MoE models (#3597)
Nick Hill committed
Mar 24, 2024 at 16:00 UTC
41deac4a3d785aa8d889acd3eebe534d060df117
4 files changed
+15
-15
tests/kernels/test_moe.py
+6
-4
@@ -81,11 +81,13 @@ def test_mixtral_moe(dtype: torch.dtype):
81
vllm_moe.w2s[i][:] = hf_moe.experts[i].w2.weight.data
82
83
# Generate input batch of dimensions [batch_size, seq_len, hidden_dim]
84
- inputs = torch.randn((1, 64, config.hidden_size)).to(dtype).to("cuda")
84
+ hf_inputs = torch.randn((1, 64, config.hidden_size)).to(dtype).to("cuda")
85
+ # vLLM uses 1D query [num_tokens, hidden_dim]
86
+ vllm_inputs = hf_inputs.flatten(0, 1)
87
88
# Run forward passes for both MoE blocks
87
- hf_states, _ = hf_moe.forward(inputs)
88
- vllm_states = vllm_moe.forward(inputs)
89
+ hf_states, _ = hf_moe.forward(hf_inputs)
90
+ vllm_states = vllm_moe.forward(vllm_inputs)
91
92
mixtral_moe_tol = {
93
torch.float32: 1e-3,
@@ -93,7 +95,7 @@ def test_mixtral_moe(dtype: torch.dtype):
95
torch.bfloat16: 1e-2,
96
}
97
96
- assert torch.allclose(hf_states,
98
+ assert torch.allclose(hf_states.flatten(0, 1),
99
vllm_states,
100
rtol=mixtral_moe_tol[dtype],
101
atol=mixtral_moe_tol[dtype])
vllm/model_executor/models/deepseek.py
+3
-4
@@ -150,11 +150,11 @@ class DeepseekMoE(nn.Module):
150
self.w2 = self.w2.view(len(w2), *w2s[0].shape)
151
152
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
153
- batch_size, sequence_length, hidden_dim = hidden_states.shape
153
+ num_tokens, hidden_dim = hidden_states.shape
154
hidden_states = hidden_states.view(-1, hidden_dim)
155
if self.config.n_shared_experts is not None:
156
shared_output = self.shared_experts(hidden_states)
157
- # router_logits: (batch * sequence_length, n_experts)
157
+ # router_logits: (num_tokens, n_experts)
158
router_logits, _ = self.gate(hidden_states)
159
final_hidden_states = fused_moe(hidden_states,
160
self.w1,
@@ -169,8 +169,7 @@ class DeepseekMoE(nn.Module):
169
final_hidden_states = tensor_model_parallel_all_reduce(
170
final_hidden_states)
171
172
- return final_hidden_states.view(batch_size, sequence_length,
173
- hidden_dim)
172
+ return final_hidden_states.view(num_tokens, hidden_dim)
173
174
175
class DeepseekAttention(nn.Module):
vllm/model_executor/models/mixtral.py
+3
-4
@@ -124,9 +124,9 @@ class MixtralMoE(nn.Module):
124
param_data[expert_id, :, :] = loaded_weight[:, shard]
125
126
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
127
- batch_size, sequence_length, hidden_size = hidden_states.shape
127
+ num_tokens, hidden_size = hidden_states.shape
128
hidden_states = hidden_states.view(-1, self.hidden_size)
129
- # router_logits: (batch * sequence_length, n_experts)
129
+ # router_logits: (num_tokens, n_experts)
130
router_logits, _ = self.gate(hidden_states)
131
final_hidden_states = fused_moe(hidden_states,
132
self.ws,
@@ -140,8 +140,7 @@ class MixtralMoE(nn.Module):
140
final_hidden_states = tensor_model_parallel_all_reduce(
141
final_hidden_states)
142
143
- return final_hidden_states.view(batch_size, sequence_length,
144
- hidden_size)
143
+ return final_hidden_states.view(num_tokens, hidden_size)
144
145
146
class MixtralAttention(nn.Module):
vllm/model_executor/models/mixtral_quant.py
+3
-3
@@ -132,9 +132,9 @@ class MixtralMoE(nn.Module):
132
linear_method=None)
133
134
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
135
- batch_size, sequence_length, hidden_dim = hidden_states.shape
135
+ num_tokens, hidden_dim = hidden_states.shape
136
hidden_states = hidden_states.view(-1, hidden_dim)
137
- # router_logits: (batch * sequence_length, n_experts)
137
+ # router_logits: (num_tokens, n_experts)
138
router_logits, _ = self.gate(hidden_states)
139
140
routing_weights = F.softmax(router_logits, dim=1, dtype=torch.float)
@@ -158,7 +158,7 @@ class MixtralMoE(nn.Module):
158
final_hidden_states.add_(current_hidden_states)
159
160
return tensor_model_parallel_all_reduce(final_hidden_states).view(
161
- batch_size, sequence_length, hidden_dim)
161
+ num_tokens, hidden_dim)
162
163
164
class MixtralAttention(nn.Module):