@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e72ae80b06
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[Bugfix] Support 2D input shape in MoE layer (#6287)

Woosuk Kwon committed Jul 10, 2024 at 06:03 UTC e72ae80b06405ea92b703c8979f046d68e970c94
2 files changed +7 -4
vllm/model_executor/models/mixtral.py
+3 -2
@@ -88,12 +88,13 @@ class MixtralMoE(nn.Module):
88 tp_size=tp_size)
89
90 def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
91 - num_tokens, hidden_size = hidden_states.shape
91 + # NOTE: hidden_states can have either 1D or 2D shape.
92 + orig_shape = hidden_states.shape
93 hidden_states = hidden_states.view(-1, self.hidden_size)
94 # router_logits: (num_tokens, n_experts)
95 router_logits, _ = self.gate(hidden_states)
96 final_hidden_states = self.experts(hidden_states, router_logits)
96 - return final_hidden_states.view(num_tokens, hidden_size)
97 + return final_hidden_states.view(orig_shape)
98
99
100 class MixtralAttention(nn.Module):
vllm/model_executor/models/qwen2_moe.py
+4 -2
@@ -126,7 +126,9 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
126 bias=False)
127
128 def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
129 - num_tokens, hidden_dim = hidden_states.shape
129 + # NOTE: hidden_states can have either 1D or 2D shape.
130 + orig_shape = hidden_states.shape
131 + hidden_dim = hidden_states.shape[-1]
132 hidden_states = hidden_states.view(-1, hidden_dim)
133 shared_output = None
134 if self.shared_expert is not None:
@@ -145,7 +147,7 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
147 final_hidden_states = tensor_model_parallel_all_reduce(
148 final_hidden_states)
149
148 - return final_hidden_states.view(num_tokens, hidden_dim)
150 + return final_hidden_states.view(orig_shape)
151
152
153 class Qwen2MoeAttention(nn.Module):