A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[dbrx] refactor dbrx experts to extend FusedMoe class (#8518)
Divakar Verma committed
Sep 21, 2024 at 16:09 UTC
9dc7c6c7f332ac6c08311c7a946c6945e0782701
1 file changed
+51
-69
vllm/model_executor/models/dbrx.py
+51
-69
@@ -7,9 +7,8 @@ import torch.nn as nn
7
from vllm.attention import Attention, AttentionMetadata
8
from vllm.config import CacheConfig
9
from vllm.distributed import (get_tensor_model_parallel_rank,
10
- get_tensor_model_parallel_world_size,
11
- tensor_model_parallel_all_reduce)
12
-from vllm.model_executor.layers.fused_moe import fused_moe
10
+ get_tensor_model_parallel_world_size)
11
+from vllm.model_executor.layers.fused_moe import FusedMoE
12
from vllm.model_executor.layers.linear import (QKVParallelLinear,
13
ReplicatedLinear,
14
RowParallelLinear)
@@ -22,7 +21,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
21
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
22
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
23
from vllm.model_executor.sampling_metadata import SamplingMetadata
25
-from vllm.model_executor.utils import set_weight_attrs
24
from vllm.sequence import IntermediateTensors
25
from vllm.transformers_utils.configs.dbrx import DbrxConfig
26
@@ -54,13 +52,7 @@ class DbrxRouter(nn.Module):
52
return router_logits
53
54
57
-class DbrxExperts(nn.Module):
58
- """A tensor-parallel MoE implementation for DBRX.
59
-
60
- Each expert's weights are sharded across all ranks and a fused MoE
61
- kernel is used for the forward pass, and finally we reduce the outputs
62
- across ranks.
63
- """
55
+class DbrxExperts(FusedMoE):
56
57
def __init__(
58
self,
@@ -68,49 +60,24 @@ class DbrxExperts(nn.Module):
60
quant_config: Optional[QuantizationConfig] = None,
61
params_dtype: Optional[torch.dtype] = None,
62
):
71
- super().__init__()
63
+ super().__init__(
64
+ num_experts=config.ffn_config.moe_num_experts,
65
+ top_k=config.ffn_config.moe_top_k,
66
+ hidden_size=config.d_model,
67
+ intermediate_size=config.ffn_config.ffn_hidden_size,
68
+ params_dtype=params_dtype,
69
+ reduce_results=True,
70
+ renormalize=True,
71
+ quant_config=quant_config,
72
+ tp_size=get_tensor_model_parallel_world_size(),
73
+ )
74
+ self.config = config
75
self.tp_size = get_tensor_model_parallel_world_size()
73
- self.num_total_experts = config.ffn_config.moe_num_experts
74
- self.top_k = config.ffn_config.moe_top_k
76
self.d_model = config.d_model
76
- self.intermediate_size = (config.ffn_config.ffn_hidden_size //
77
+ self.intermediate_size = (self.config.ffn_config.ffn_hidden_size //
78
self.tp_size)
79
79
- if params_dtype is None:
80
- params_dtype = torch.get_default_dtype()
81
- self.params_dtype = params_dtype
82
-
83
- self.router = DbrxRouter(config, self.params_dtype)
84
- self.ws = nn.Parameter(
85
- torch.empty(
86
- self.num_total_experts,
87
- 2 * self.intermediate_size,
88
- self.d_model,
89
- device="cuda",
90
- dtype=self.params_dtype,
91
- ))
92
- self.w2s = nn.Parameter(
93
- torch.empty(
94
- self.num_total_experts,
95
- self.d_model,
96
- self.intermediate_size,
97
- device="cuda",
98
- dtype=self.params_dtype,
99
- ))
100
-
101
- set_weight_attrs(
102
- self.ws,
103
- {
104
- "weight_loader": self.weight_loader,
105
- },
106
- )
107
- set_weight_attrs(
108
- self.w2s,
109
- {
110
- "weight_loader": self.weight_loader,
111
- },
112
- )
113
-
80
+ # Define custom weight loader for dbrx model
81
def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor,
82
weight_name: str):
83
tp_rank = get_tensor_model_parallel_rank()
@@ -140,26 +107,40 @@ class DbrxExperts(nn.Module):
107
).transpose(1, 2)
108
param_data[:] = loaded_weight[:, :, shard]
109
110
+
111
+class DbrxMoE(nn.Module):
112
+ """A tensor-parallel MoE implementation for DBRX.
113
+
114
+ Each expert's weights are sharded across all ranks and a fused MoE
115
+ kernel is used for the forward pass, and finally we reduce the outputs
116
+ across ranks.
117
+ """
118
+
119
+ def __init__(
120
+ self,
121
+ config: DbrxConfig,
122
+ quant_config: Optional[QuantizationConfig] = None,
123
+ params_dtype: Optional[torch.dtype] = None,
124
+ ):
125
+ super().__init__()
126
+ self.d_model = config.d_model
127
+ if params_dtype is None:
128
+ params_dtype = torch.get_default_dtype()
129
+ self.params_dtype = params_dtype
130
+
131
+ self.router = DbrxRouter(config, self.params_dtype)
132
+
133
+ self.experts = DbrxExperts(config=config,
134
+ quant_config=quant_config,
135
+ params_dtype=self.params_dtype)
136
+
137
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
144
- num_tokens, hidden_size = hidden_states.shape
138
+ orig_shape = hidden_states.shape
139
hidden_states = hidden_states.view(-1, self.d_model)
140
# router_logits: (num_tokens, n_experts)
141
router_logits = self.router(hidden_states)
148
- final_hidden_states = fused_moe(
149
- hidden_states,
150
- self.ws,
151
- self.w2s,
152
- router_logits,
153
- self.top_k,
154
- renormalize=True,
155
- inplace=True,
156
- )
157
-
158
- if self.tp_size > 1:
159
- final_hidden_states = tensor_model_parallel_all_reduce(
160
- final_hidden_states)
161
-
162
- return final_hidden_states.view(num_tokens, hidden_size)
142
+ final_hidden_states = self.experts(hidden_states, router_logits)
143
+ return final_hidden_states.view(orig_shape)
144
145
146
class DbrxAttention(nn.Module):
@@ -288,7 +269,7 @@ class DbrxBlock(nn.Module):
269
super().__init__()
270
self.norm_attn_norm = DbrxFusedNormAttention(config, cache_config,
271
quant_config)
291
- self.ffn = DbrxExperts(config, quant_config)
272
+ self.ffn = DbrxMoE(config, quant_config)
273
274
def forward(
275
self,
@@ -409,9 +390,10 @@ class DbrxForCausalLM(nn.Module):
390
return next_tokens
391
392
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
393
+
394
expert_params_mapping = [(
413
- "ws" if weight_name in ["w1", "v1"] else "w2s",
414
- f"experts.mlp.{weight_name}",
395
+ "w13_weight" if weight_name in ["w1", "v1"] else "w2_weight",
396
+ f"mlp.{weight_name}",
397
) for weight_name in ["w1", "v1", "w2"]]
398
params_dict = dict(self.named_parameters(remove_duplicate=False))
399
for name, loaded_weight in weights: