@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ab40644669
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Fused MOE for Mixtral (#2542)

Co-authored-by: chen shen <scv119@gmail.com>

Philipp Moritz committed Jan 29, 2024 at 22:43 UTC ab406446691f289ef51d1abd8d1ff66760eda36f
4 files changed +115 -109
csrc/moe_align_block_size_kernels.cu
+1 -1
@@ -95,7 +95,7 @@ void moe_align_block_size(
95 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
96 assert(num_experts <= NUM_MAX_EXPERTS);
97 VLLM_DISPATCH_INTEGRAL_TYPES(
98 - topk_ids.scalar_type(), "moe_alig_block_size_kernel", [&] {
98 + topk_ids.scalar_type(), "moe_align_block_size_kernel", [&] {
99 vllm::moe_align_block_size_kernel<scalar_t><<<1, num_experts, 0, stream>>>(
100 topk_ids.data_ptr<scalar_t>(),
101 sorted_token_ids.data_ptr<int32_t>(),
csrc/ops.h
+7 -9
@@ -100,6 +100,13 @@ void gptq_shuffle(
100 torch::Tensor q_weight,
101 torch::Tensor q_perm);
102
103 +void moe_align_block_size(
104 + torch::Tensor topk_ids,
105 + int num_experts,
106 + int block_size,
107 + torch::Tensor sorted_token_ids,
108 + torch::Tensor experts_ids,
109 + torch::Tensor num_tokens_post_pad);
110
111 #ifndef USE_ROCM
112 using fptr_t = uint64_t;
@@ -121,12 +128,3 @@ std::pair<std::vector<uint8_t>, std::vector<int64_t>> get_graph_buffer_ipc_meta(
128 void register_graph_buffers(fptr_t _fa, const std::vector<std::string> &handles,
129 const std::vector<std::vector<int64_t>> &offsets);
130 #endif
124 -
125 -void moe_align_block_size(
126 - torch::Tensor topk_ids,
127 - int num_experts,
128 - int block_size,
129 - torch::Tensor sorted_token_ids,
130 - torch::Tensor experts_ids,
131 - torch::Tensor num_tokens_post_pad
132 - );
csrc/pybind.cpp
+3 -3
@@ -57,9 +57,9 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
57 ops.def("gptq_shuffle", &gptq_shuffle, "Post processing for GPTQ");
58 ops.def("squeezellm_gemm", &squeezellm_gemm, "Quantized GEMM for SqueezeLLM");
59 ops.def(
60 - "moe_align_block_size",
61 - &moe_align_block_size,
62 - "Aligning the number of tokens to be processed by each expert such that it is divisible by the block size.");
60 + "moe_align_block_size",
61 + &moe_align_block_size,
62 + "Aligning the number of tokens to be processed by each expert such that it is divisible by the block size.");
63
64 // Cache ops
65 pybind11::module cache_ops = m.def_submodule("cache_ops", "vLLM cache ops");
vllm/model_executor/models/mixtral.py
+104 -96
@@ -23,8 +23,6 @@
23 """Inference-only Mixtral model."""
24 from typing import List, Optional, Tuple
25
26 -import numpy as np
27 -
26 import torch
27 import torch.nn.functional as F
28
@@ -33,10 +31,11 @@ from transformers import MixtralConfig
31
32 from vllm.model_executor.input_metadata import InputMetadata
33 from vllm.model_executor.layers.attention import PagedAttention
34 +from vllm.model_executor.layers.fused_moe import fused_moe
35 from vllm.model_executor.layers.layernorm import RMSNorm
36 from vllm.model_executor.layers.linear import (LinearMethodBase,
38 - ReplicatedLinear,
37 QKVParallelLinear,
38 + ReplicatedLinear,
39 RowParallelLinear)
40 from vllm.model_executor.layers.rotary_embedding import get_rope
41 from vllm.model_executor.layers.sampler import Sampler
@@ -47,6 +46,7 @@ from vllm.model_executor.parallel_utils.communication_op import (
46 from vllm.model_executor.parallel_utils.parallel_state import (
47 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
48 from vllm.model_executor.sampling_metadata import SamplingMetadata
49 +from vllm.model_executor.utils import set_weight_attrs
50 from vllm.model_executor.weight_utils import (default_weight_loader,
51 hf_model_weights_iterator)
52 from vllm.sequence import SamplerOutput
@@ -54,85 +54,77 @@ from vllm.sequence import SamplerOutput
54 KVCache = Tuple[torch.Tensor, torch.Tensor]
55
56
57 -class MixtralMLP(nn.Module):
57 +class MixtralMoE(nn.Module):
58 + """A tensor-parallel MoE implementation for Mixtral that shards each expert
59 + across all ranks.
60 +
61 + Each expert's weights are sharded across all ranks and a fused MoE
62 + kernel is used for the forward pass, and finally we reduce the outputs
63 + across ranks.
64 + """
65
66 def __init__(
67 self,
68 num_experts: int,
69 + top_k: int,
70 hidden_size: int,
71 intermediate_size: int,
64 - linear_method: Optional[LinearMethodBase] = None,
65 - ) -> None:
72 + params_dtype: Optional[torch.dtype] = None,
73 + ):
74 super().__init__()
67 - self.num_experts = num_experts
68 - self.ffn_dim = intermediate_size
69 - self.hidden_dim = hidden_size
70 -
71 - self.w1 = ReplicatedLinear(self.hidden_dim,
72 - self.ffn_dim,
73 - bias=False,
74 - linear_method=linear_method)
75 - self.w2 = ReplicatedLinear(self.ffn_dim,
76 - self.hidden_dim,
77 - bias=False,
78 - linear_method=linear_method)
79 - self.w3 = ReplicatedLinear(self.hidden_dim,
80 - self.ffn_dim,
81 - bias=False,
82 - linear_method=linear_method)
83 -
84 - # TODO: Use vllm's SiluAndMul
85 - self.act_fn = nn.SiLU()
86 -
87 - def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
88 - w1_out, _ = self.w1(hidden_states)
89 - w1_out = self.act_fn(w1_out)
90 - w3_out, _ = self.w3(hidden_states)
91 - current_hidden_states = w1_out * w3_out
92 - current_hidden_states, _ = self.w2(current_hidden_states)
93 - return current_hidden_states
94 -
75 + tp_size = get_tensor_model_parallel_world_size()
76 + self.num_total_experts = num_experts
77 + self.top_k = top_k
78 + self.hidden_size = hidden_size
79 + self.intermediate_size = intermediate_size // tp_size
80
96 -class MixtralMoE(nn.Module):
81 + if params_dtype is None:
82 + params_dtype = torch.get_default_dtype()
83 + self.params_dtype = params_dtype
84
98 - def __init__(
99 - self,
100 - config: MixtralConfig,
101 - linear_method: Optional[LinearMethodBase] = None,
102 - ):
103 - super().__init__()
104 - self.config = config
105 - self.rank = get_tensor_model_parallel_rank()
106 - self.tp_size = get_tensor_model_parallel_world_size()
107 - self.num_total_experts = config.num_local_experts
108 - self.top_k = config.num_experts_per_tok
109 - if self.tp_size > self.num_total_experts:
110 - raise ValueError(
111 - f"Tensor parallel size {self.tp_size} is greater than "
112 - f"the number of experts {self.num_total_experts}.")
113 - # Split experts equally between ranks
114 - self.expert_indicies = np.array_split(range(
115 - self.num_total_experts), self.tp_size)[self.rank].tolist()
116 - if not self.expert_indicies:
117 - raise ValueError(
118 - f"Rank {self.rank} has no experts assigned to it.")
119 -
120 - self.experts = nn.ModuleList([
121 - MixtralMLP(self.num_total_experts,
122 - config.hidden_size,
123 - config.intermediate_size,
124 - linear_method=linear_method)
125 - if idx in self.expert_indicies else None
126 - for idx in range(self.num_total_experts)
127 - ])
128 - self.gate = ReplicatedLinear(config.hidden_size,
85 + self.gate = ReplicatedLinear(self.hidden_size,
86 self.num_total_experts,
87 bias=False,
88 + params_dtype=self.params_dtype,
89 linear_method=None)
90
91 + self.ws = nn.Parameter(
92 + torch.empty(self.num_total_experts,
93 + 2 * self.intermediate_size,
94 + self.hidden_size,
95 + device="cuda",
96 + dtype=self.params_dtype))
97 + self.w2s = nn.Parameter(
98 + torch.empty(self.num_total_experts,
99 + self.hidden_size,
100 + self.intermediate_size,
101 + device="cuda",
102 + dtype=self.params_dtype))
103 +
104 + set_weight_attrs(self.ws, {
105 + "weight_loader": self.weight_loader,
106 + })
107 + set_weight_attrs(self.w2s, {
108 + "weight_loader": self.weight_loader,
109 + })
110 +
111 + def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor,
112 + weight_name: str, expert_id: int):
113 + tp_rank = get_tensor_model_parallel_rank()
114 + param_data = param.data
115 + shard_size = self.intermediate_size
116 + shard = slice(tp_rank * shard_size, (tp_rank + 1) * shard_size)
117 + if weight_name.endswith("w1.weight"):
118 + param_data[expert_id, 0:shard_size, :] = loaded_weight[shard, :]
119 + if weight_name.endswith("w3.weight"):
120 + param_data[expert_id,
121 + shard_size:2 * shard_size, :] = loaded_weight[shard, :]
122 + if weight_name.endswith("w2.weight"):
123 + param_data[expert_id, :, :] = loaded_weight[:, shard]
124 +
125 def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
134 - batch_size, sequence_length, hidden_dim = hidden_states.shape
135 - hidden_states = hidden_states.view(-1, hidden_dim)
126 + batch_size, sequence_length, hidden_size = hidden_states.shape
127 + hidden_states = hidden_states.view(-1, self.hidden_size)
128 # router_logits: (batch * sequence_length, n_experts)
129 router_logits, _ = self.gate(hidden_states)
130
@@ -142,22 +134,18 @@ class MixtralMoE(nn.Module):
134 dim=-1)
135 routing_weights /= routing_weights.sum(dim=-1, keepdim=True)
136
145 - final_hidden_states = None
146 - for expert_idx in self.expert_indicies:
147 - expert_layer = self.experts[expert_idx]
148 - expert_mask = (selected_experts == expert_idx)
149 - expert_weights = (routing_weights * expert_mask).sum(dim=-1,
150 - keepdim=True)
151 -
152 - current_hidden_states = expert_layer(hidden_states).mul_(
153 - expert_weights)
154 - if final_hidden_states is None:
155 - final_hidden_states = current_hidden_states
156 - else:
157 - final_hidden_states.add_(current_hidden_states)
137 + final_hidden_states = fused_moe(hidden_states,
138 + self.ws,
139 + self.w2s,
140 + routing_weights,
141 + selected_experts,
142 + inplace=True)
143 +
144 + final_hidden_states = tensor_model_parallel_all_reduce(
145 + final_hidden_states)
146
159 - return tensor_model_parallel_all_reduce(final_hidden_states).view(
160 - batch_size, sequence_length, hidden_dim)
147 + return final_hidden_states.view(batch_size, sequence_length,
148 + hidden_size)
149
150
151 class MixtralAttention(nn.Module):
@@ -257,8 +245,11 @@ class MixtralDecoderLayer(nn.Module):
245 rope_theta=rope_theta,
246 sliding_window=config.sliding_window,
247 linear_method=linear_method)
260 - self.block_sparse_moe = MixtralMoE(config=config,
261 - linear_method=linear_method)
248 + self.block_sparse_moe = MixtralMoE(
249 + num_experts=config.num_local_experts,
250 + top_k=config.num_experts_per_tok,
251 + hidden_size=config.hidden_size,
252 + intermediate_size=config.intermediate_size)
253 self.input_layernorm = RMSNorm(config.hidden_size,
254 eps=config.rms_norm_eps)
255 self.post_attention_layernorm = RMSNorm(config.hidden_size,
@@ -378,6 +369,14 @@ class MixtralForCausalLM(nn.Module):
369 ("qkv_proj", "v_proj", "v"),
370 ]
371
372 + expert_params_mapping = [
373 + # (param_name, weight_name, expert_id)
374 + ("ws" if weight_name in ["w1", "w3"] else "w2s",
375 + f"experts.{expert_id}.{weight_name}.weight", expert_id)
376 + for expert_id in range(self.config.num_local_experts)
377 + for weight_name in ["w1", "w2", "w3"]
378 + ]
379 +
380 params_dict = dict(self.named_parameters())
381 for name, loaded_weight in hf_model_weights_iterator(
382 model_name_or_path,
@@ -387,6 +386,7 @@ class MixtralForCausalLM(nn.Module):
386 fall_back_to_pt=False):
387 if "rotary_emb.inv_freq" in name:
388 continue
389 +
390 for (param_name, weight_name, shard_id) in stacked_params_mapping:
391 if weight_name not in name:
392 continue
@@ -399,14 +399,22 @@ class MixtralForCausalLM(nn.Module):
399 weight_loader(param, loaded_weight, shard_id)
400 break
401 else:
402 - # Skip loading extra bias for GPTQ models.
403 - if name.endswith(".bias") and name not in params_dict:
404 - continue
405 - # Skip experts that are not assigned to this worker.
406 - if ("block_sparse_moe.experts." in name
407 - and name not in params_dict):
408 - continue
409 - param = params_dict[name]
410 - weight_loader = getattr(param, "weight_loader",
411 - default_weight_loader)
412 - weight_loader(param, loaded_weight)
402 + for param_name, weight_name, expert_id in expert_params_mapping:
403 + if weight_name not in name:
404 + continue
405 + name = name.replace(weight_name, param_name)
406 + param = params_dict[name]
407 + weight_loader = param.weight_loader
408 + weight_loader(param,
409 + loaded_weight,
410 + weight_name,
411 + expert_id=expert_id)
412 + break
413 + else:
414 + # Skip loading extra bias for GPTQ models.
415 + if name.endswith(".bias") and name not in params_dict:
416 + continue
417 + param = params_dict[name]
418 + weight_loader = getattr(param, "weight_loader",
419 + default_weight_loader)
420 + weight_loader(param, loaded_weight)