@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9dc7c6c7f3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[dbrx] refactor dbrx experts to extend FusedMoe class (#8518)

Divakar Verma committed Sep 21, 2024 at 16:09 UTC 9dc7c6c7f332ac6c08311c7a946c6945e0782701
1 file changed +51 -69
vllm/model_executor/models/dbrx.py
+51 -69
@@ -7,9 +7,8 @@ import torch.nn as nn
7 from vllm.attention import Attention, AttentionMetadata
8 from vllm.config import CacheConfig
9 from vllm.distributed import (get_tensor_model_parallel_rank,
10 - get_tensor_model_parallel_world_size,
11 - tensor_model_parallel_all_reduce)
12 -from vllm.model_executor.layers.fused_moe import fused_moe
10 + get_tensor_model_parallel_world_size)
11 +from vllm.model_executor.layers.fused_moe import FusedMoE
12 from vllm.model_executor.layers.linear import (QKVParallelLinear,
13 ReplicatedLinear,
14 RowParallelLinear)
@@ -22,7 +21,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
21 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
22 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
23 from vllm.model_executor.sampling_metadata import SamplingMetadata
25 -from vllm.model_executor.utils import set_weight_attrs
24 from vllm.sequence import IntermediateTensors
25 from vllm.transformers_utils.configs.dbrx import DbrxConfig
26
@@ -54,13 +52,7 @@ class DbrxRouter(nn.Module):
52 return router_logits
53
54
57 -class DbrxExperts(nn.Module):
58 - """A tensor-parallel MoE implementation for DBRX.
59 -
60 - Each expert's weights are sharded across all ranks and a fused MoE
61 - kernel is used for the forward pass, and finally we reduce the outputs
62 - across ranks.
63 - """
55 +class DbrxExperts(FusedMoE):
56
57 def __init__(
58 self,
@@ -68,49 +60,24 @@ class DbrxExperts(nn.Module):
60 quant_config: Optional[QuantizationConfig] = None,
61 params_dtype: Optional[torch.dtype] = None,
62 ):
71 - super().__init__()
63 + super().__init__(
64 + num_experts=config.ffn_config.moe_num_experts,
65 + top_k=config.ffn_config.moe_top_k,
66 + hidden_size=config.d_model,
67 + intermediate_size=config.ffn_config.ffn_hidden_size,
68 + params_dtype=params_dtype,
69 + reduce_results=True,
70 + renormalize=True,
71 + quant_config=quant_config,
72 + tp_size=get_tensor_model_parallel_world_size(),
73 + )
74 + self.config = config
75 self.tp_size = get_tensor_model_parallel_world_size()
73 - self.num_total_experts = config.ffn_config.moe_num_experts
74 - self.top_k = config.ffn_config.moe_top_k
76 self.d_model = config.d_model
76 - self.intermediate_size = (config.ffn_config.ffn_hidden_size //
77 + self.intermediate_size = (self.config.ffn_config.ffn_hidden_size //
78 self.tp_size)
79
79 - if params_dtype is None:
80 - params_dtype = torch.get_default_dtype()
81 - self.params_dtype = params_dtype
82 -
83 - self.router = DbrxRouter(config, self.params_dtype)
84 - self.ws = nn.Parameter(
85 - torch.empty(
86 - self.num_total_experts,
87 - 2 * self.intermediate_size,
88 - self.d_model,
89 - device="cuda",
90 - dtype=self.params_dtype,
91 - ))
92 - self.w2s = nn.Parameter(
93 - torch.empty(
94 - self.num_total_experts,
95 - self.d_model,
96 - self.intermediate_size,
97 - device="cuda",
98 - dtype=self.params_dtype,
99 - ))
100 -
101 - set_weight_attrs(
102 - self.ws,
103 - {
104 - "weight_loader": self.weight_loader,
105 - },
106 - )
107 - set_weight_attrs(
108 - self.w2s,
109 - {
110 - "weight_loader": self.weight_loader,
111 - },
112 - )
113 -
80 + # Define custom weight loader for dbrx model
81 def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor,
82 weight_name: str):
83 tp_rank = get_tensor_model_parallel_rank()
@@ -140,26 +107,40 @@ class DbrxExperts(nn.Module):
107 ).transpose(1, 2)
108 param_data[:] = loaded_weight[:, :, shard]
109
110 +
111 +class DbrxMoE(nn.Module):
112 + """A tensor-parallel MoE implementation for DBRX.
113 +
114 + Each expert's weights are sharded across all ranks and a fused MoE
115 + kernel is used for the forward pass, and finally we reduce the outputs
116 + across ranks.
117 + """
118 +
119 + def __init__(
120 + self,
121 + config: DbrxConfig,
122 + quant_config: Optional[QuantizationConfig] = None,
123 + params_dtype: Optional[torch.dtype] = None,
124 + ):
125 + super().__init__()
126 + self.d_model = config.d_model
127 + if params_dtype is None:
128 + params_dtype = torch.get_default_dtype()
129 + self.params_dtype = params_dtype
130 +
131 + self.router = DbrxRouter(config, self.params_dtype)
132 +
133 + self.experts = DbrxExperts(config=config,
134 + quant_config=quant_config,
135 + params_dtype=self.params_dtype)
136 +
137 def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
144 - num_tokens, hidden_size = hidden_states.shape
138 + orig_shape = hidden_states.shape
139 hidden_states = hidden_states.view(-1, self.d_model)
140 # router_logits: (num_tokens, n_experts)
141 router_logits = self.router(hidden_states)
148 - final_hidden_states = fused_moe(
149 - hidden_states,
150 - self.ws,
151 - self.w2s,
152 - router_logits,
153 - self.top_k,
154 - renormalize=True,
155 - inplace=True,
156 - )
157 -
158 - if self.tp_size > 1:
159 - final_hidden_states = tensor_model_parallel_all_reduce(
160 - final_hidden_states)
161 -
162 - return final_hidden_states.view(num_tokens, hidden_size)
142 + final_hidden_states = self.experts(hidden_states, router_logits)
143 + return final_hidden_states.view(orig_shape)
144
145
146 class DbrxAttention(nn.Module):
@@ -288,7 +269,7 @@ class DbrxBlock(nn.Module):
269 super().__init__()
270 self.norm_attn_norm = DbrxFusedNormAttention(config, cache_config,
271 quant_config)
291 - self.ffn = DbrxExperts(config, quant_config)
272 + self.ffn = DbrxMoE(config, quant_config)
273
274 def forward(
275 self,
@@ -409,9 +390,10 @@ class DbrxForCausalLM(nn.Module):
390 return next_tokens
391
392 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
393 +
394 expert_params_mapping = [(
413 - "ws" if weight_name in ["w1", "v1"] else "w2s",
414 - f"experts.mlp.{weight_name}",
395 + "w13_weight" if weight_name in ["w1", "v1"] else "w2_weight",
396 + f"mlp.{weight_name}",
397 ) for weight_name in ["w1", "v1", "w2"]]
398 params_dict = dict(self.named_parameters(remove_duplicate=False))
399 for name, loaded_weight in weights: