@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 41deac4a3d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[BugFix] 1D query fix for MoE models (#3597)

Nick Hill committed Mar 24, 2024 at 16:00 UTC 41deac4a3d785aa8d889acd3eebe534d060df117
4 files changed +15 -15
tests/kernels/test_moe.py
+6 -4
@@ -81,11 +81,13 @@ def test_mixtral_moe(dtype: torch.dtype):
81 vllm_moe.w2s[i][:] = hf_moe.experts[i].w2.weight.data
82
83 # Generate input batch of dimensions [batch_size, seq_len, hidden_dim]
84 - inputs = torch.randn((1, 64, config.hidden_size)).to(dtype).to("cuda")
84 + hf_inputs = torch.randn((1, 64, config.hidden_size)).to(dtype).to("cuda")
85 + # vLLM uses 1D query [num_tokens, hidden_dim]
86 + vllm_inputs = hf_inputs.flatten(0, 1)
87
88 # Run forward passes for both MoE blocks
87 - hf_states, _ = hf_moe.forward(inputs)
88 - vllm_states = vllm_moe.forward(inputs)
89 + hf_states, _ = hf_moe.forward(hf_inputs)
90 + vllm_states = vllm_moe.forward(vllm_inputs)
91
92 mixtral_moe_tol = {
93 torch.float32: 1e-3,
@@ -93,7 +95,7 @@ def test_mixtral_moe(dtype: torch.dtype):
95 torch.bfloat16: 1e-2,
96 }
97
96 - assert torch.allclose(hf_states,
98 + assert torch.allclose(hf_states.flatten(0, 1),
99 vllm_states,
100 rtol=mixtral_moe_tol[dtype],
101 atol=mixtral_moe_tol[dtype])
vllm/model_executor/models/deepseek.py
+3 -4
@@ -150,11 +150,11 @@ class DeepseekMoE(nn.Module):
150 self.w2 = self.w2.view(len(w2), *w2s[0].shape)
151
152 def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
153 - batch_size, sequence_length, hidden_dim = hidden_states.shape
153 + num_tokens, hidden_dim = hidden_states.shape
154 hidden_states = hidden_states.view(-1, hidden_dim)
155 if self.config.n_shared_experts is not None:
156 shared_output = self.shared_experts(hidden_states)
157 - # router_logits: (batch * sequence_length, n_experts)
157 + # router_logits: (num_tokens, n_experts)
158 router_logits, _ = self.gate(hidden_states)
159 final_hidden_states = fused_moe(hidden_states,
160 self.w1,
@@ -169,8 +169,7 @@ class DeepseekMoE(nn.Module):
169 final_hidden_states = tensor_model_parallel_all_reduce(
170 final_hidden_states)
171
172 - return final_hidden_states.view(batch_size, sequence_length,
173 - hidden_dim)
172 + return final_hidden_states.view(num_tokens, hidden_dim)
173
174
175 class DeepseekAttention(nn.Module):
vllm/model_executor/models/mixtral.py
+3 -4
@@ -124,9 +124,9 @@ class MixtralMoE(nn.Module):
124 param_data[expert_id, :, :] = loaded_weight[:, shard]
125
126 def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
127 - batch_size, sequence_length, hidden_size = hidden_states.shape
127 + num_tokens, hidden_size = hidden_states.shape
128 hidden_states = hidden_states.view(-1, self.hidden_size)
129 - # router_logits: (batch * sequence_length, n_experts)
129 + # router_logits: (num_tokens, n_experts)
130 router_logits, _ = self.gate(hidden_states)
131 final_hidden_states = fused_moe(hidden_states,
132 self.ws,
@@ -140,8 +140,7 @@ class MixtralMoE(nn.Module):
140 final_hidden_states = tensor_model_parallel_all_reduce(
141 final_hidden_states)
142
143 - return final_hidden_states.view(batch_size, sequence_length,
144 - hidden_size)
143 + return final_hidden_states.view(num_tokens, hidden_size)
144
145
146 class MixtralAttention(nn.Module):
vllm/model_executor/models/mixtral_quant.py
+3 -3
@@ -132,9 +132,9 @@ class MixtralMoE(nn.Module):
132 linear_method=None)
133
134 def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
135 - batch_size, sequence_length, hidden_dim = hidden_states.shape
135 + num_tokens, hidden_dim = hidden_states.shape
136 hidden_states = hidden_states.view(-1, hidden_dim)
137 - # router_logits: (batch * sequence_length, n_experts)
137 + # router_logits: (num_tokens, n_experts)
138 router_logits, _ = self.gate(hidden_states)
139
140 routing_weights = F.softmax(router_logits, dim=1, dtype=torch.float)
@@ -158,7 +158,7 @@ class MixtralMoE(nn.Module):
158 final_hidden_states.add_(current_hidden_states)
159
160 return tensor_model_parallel_all_reduce(final_hidden_states).view(
161 - batch_size, sequence_length, hidden_dim)
161 + num_tokens, hidden_dim)
162
163
164 class MixtralAttention(nn.Module):