A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Optimize Mixtral with expert parallelism (#2090)
Antoni Baum committed
Dec 13, 2023 at 23:55 UTC
21d93c140d0a97af5f0c59e660cf04bd417fd424
6 files changed
+221
-334
Dockerfile
+1
-13
@@ -41,14 +41,6 @@ ENV NVCC_THREADS=$nvcc_threads
41
42
RUN python3 setup.py build_ext --inplace
43
44
-# Build the megablocks library as wheel because it doesn't publish pre-built wheels.
45
-# https://github.com/stanford-futuredata/megablocks/commit/5897cd6f254b7b3edf7a708a3a3314ecb54b6f78
46
-RUN apt-get install -y git && \
47
- git clone https://github.com/stanford-futuredata/megablocks.git && \
48
- cd megablocks && \
49
- git checkout 5897cd6f254b7b3edf7a708a3a3314ecb54b6f78 && \
50
- MAX_JOBS=8 NVCC_THREADS=8 python3 setup.py bdist_wheel
51
-
44
# image to run unit testing suite
45
FROM dev AS test
46
@@ -85,12 +77,8 @@ FROM vllm-base AS vllm-openai
77
RUN --mount=type=cache,target=/root/.cache/pip \
78
pip install accelerate
79
88
-COPY vllm vllm
80
COPY --from=build /workspace/vllm/*.so /workspace/vllm/
90
-COPY --from=build /workspace/megablocks/dist/*.whl /tmp/
91
-RUN --mount=type=cache,target=/root/.cache/pip \
92
- pip install /tmp/megablocks-0.5.0-cp310-cp310-linux_x86_64.whl && \
93
- rm /tmp/megablocks-0.5.0-cp310-cp310-linux_x86_64.whl
81
+COPY vllm vllm
82
83
ENTRYPOINT ["python3", "-m", "vllm.entrypoints.openai.api_server"]
84
README.md
-4
@@ -72,10 +72,6 @@ Install vLLM with pip or [from source](https://vllm.readthedocs.io/en/latest/get
72
```bash
73
pip install vllm
74
```
75
-**NOTE:** The Mixtral model additionally requires `megablocks` which can be installed with pip or [from source](https://github.com/stanford-futuredata/megablocks):
76
-```bash
77
-pip install megablocks
78
-```
75
76
## Getting Started
77
docs/source/models/supported_models.rst
+1
-2
@@ -74,8 +74,7 @@ Otherwise, please refer to :ref:`Adding a New Model <adding_a_new_model>` for in
74
Alternatively, you can raise an issue on our `GitHub <https://github.com/vllm-project/vllm/issues>`_ project.
75
76
.. note::
77
- Currently, the ROCm version of vLLM does not support Mixtral.
78
- Additionally, it only supports Mistral for context lengths up to 4096.
77
+ Currently, the ROCm version of vLLM supports Mistral and Mixtral only for context lengths up to 4096.
78
79
.. tip::
80
The easiest way to check if your model is supported is to run the program below:
vllm/config.py
+9
-7
@@ -120,14 +120,16 @@ class ModelConfig:
120
if load_format == "auto":
121
load_format = "pt"
122
123
- # FIXME(woosuk): This is a temporary hack. Support safetensor weights.
123
+ # TODO: Remove this check once HF updates the pt weights of Mixtral.
124
architectures = getattr(self.hf_config, "architectures", [])
125
- if "MixtralForCausalLM" in architectures and load_format != "pt":
126
- logger.info(
127
- "Currently, only 'pt' format is supported for Mixtral. "
128
- "Changing the format to 'pt'. This may re-download the "
129
- "weights if you have downloaded the safetensor weights.")
130
- load_format = "pt"
125
+ if "MixtralForCausalLM" in architectures:
126
+ if load_format == "pt":
127
+ raise ValueError(
128
+ "Currently, the 'pt' format is not supported for Mixtral. "
129
+ "Please use the 'safetensors' format instead. ")
130
+ elif load_format == "auto":
131
+ # Do not fall back to pt weights.
132
+ load_format = "safetensors"
133
134
self.load_format = load_format
135
vllm/model_executor/models/__init__.py
+3
-1
@@ -39,13 +39,15 @@ _MODELS = {
39
}
40
41
# Models not supported by ROCm.
42
-_ROCM_UNSUPPORTED_MODELS = ["MixtralForCausalLM"]
42
+_ROCM_UNSUPPORTED_MODELS = []
43
44
# Models partially supported by ROCm.
45
# Architecture -> Reason.
46
_ROCM_PARTIALLY_SUPPORTED_MODELS = {
47
"MistralForCausalLM":
48
"Sliding window attention is not yet supported in ROCm's flash attention",
49
+ "MixtralForCausalLM":
50
+ "Sliding window attention is not yet supported in ROCm's flash attention",
51
}
52
53
vllm/model_executor/models/mixtral.py
+207
-307
@@ -31,22 +31,11 @@ import torch.nn.functional as F
31
from torch import nn
32
from transformers import MixtralConfig
33
34
-try:
35
- import megablocks.ops as ops
36
-except ImportError as e:
37
- raise ImportError("MegaBlocks not found. "
38
- "Please install it by `pip install megablocks`.") from e
39
-try:
40
- import stk
41
-except ImportError as e:
42
- raise ImportError(
43
- "STK not found. "
44
- "Please install it by `pip install stanford-stk`.") from e
45
-
34
from vllm.model_executor.input_metadata import InputMetadata
35
from vllm.model_executor.layers.attention import PagedAttention
36
from vllm.model_executor.layers.layernorm import RMSNorm
37
from vllm.model_executor.layers.linear import (LinearMethodBase,
38
+ ReplicatedLinear,
39
QKVParallelLinear,
40
RowParallelLinear)
41
from vllm.model_executor.layers.rotary_embedding import get_rope
@@ -66,8 +55,134 @@ from vllm.sequence import SamplerOutput
55
KVCache = Tuple[torch.Tensor, torch.Tensor]
56
57
69
-def promote_scalar(x: torch.Tensor) -> torch.Tensor:
70
- return x.view(1) if len(x.size()) == 0 else x
58
+class MixtralMLP(nn.Module):
59
+
60
+ def __init__(
61
+ self,
62
+ num_experts: int,
63
+ hidden_size: int,
64
+ intermediate_size: int,
65
+ linear_method: Optional[LinearMethodBase] = None,
66
+ ) -> None:
67
+ super().__init__()
68
+ self.num_experts = num_experts
69
+ self.ffn_dim = intermediate_size
70
+ self.hidden_dim = hidden_size
71
+
72
+ self.w1 = ReplicatedLinear(self.hidden_dim,
73
+ self.ffn_dim,
74
+ bias=False,
75
+ linear_method=linear_method)
76
+ self.w2 = ReplicatedLinear(self.ffn_dim,
77
+ self.hidden_dim,
78
+ bias=False,
79
+ linear_method=linear_method)
80
+ self.w3 = ReplicatedLinear(self.hidden_dim,
81
+ self.ffn_dim,
82
+ bias=False,
83
+ linear_method=linear_method)
84
+
85
+ # TODO: Use vllm's SiluAndMul
86
+ self.act_fn = nn.SiLU()
87
+
88
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
89
+ w1_out, _ = self.w1(hidden_states)
90
+ w1_out = self.act_fn(w1_out)
91
+ w3_out, _ = self.w3(hidden_states)
92
+ current_hidden_states = w1_out * w3_out
93
+ current_hidden_states, _ = self.w2(current_hidden_states)
94
+ return current_hidden_states
95
+
96
+
97
+class DummyModule(nn.Module):
98
+
99
+ def __init__(self) -> None:
100
+ super().__init__()
101
+
102
+ self.w1 = nn.Linear(0, 0, bias=False)
103
+ self.w2 = nn.Linear(0, 0, bias=False)
104
+ self.w3 = nn.Linear(0, 0, bias=False)
105
+
106
+ set_weight_attrs(self.w1.weight,
107
+ {"weight_loader": self.dummy_weight_loader})
108
+ set_weight_attrs(self.w2.weight,
109
+ {"weight_loader": self.dummy_weight_loader})
110
+ set_weight_attrs(self.w3.weight,
111
+ {"weight_loader": self.dummy_weight_loader})
112
+
113
+ def forward(self, *args, **kwargs) -> None:
114
+ raise NotImplementedError()
115
+
116
+ def dummy_weight_loader(self, *args, **kwargs) -> None: # pylint: disable=unused-argument
117
+ # Noop
118
+ return
119
+
120
+
121
+class MixtralMoE(nn.Module):
122
+
123
+ def __init__(
124
+ self,
125
+ config: MixtralConfig,
126
+ linear_method: Optional[LinearMethodBase] = None,
127
+ ):
128
+ super().__init__()
129
+ self.config = config
130
+ self.rank = get_tensor_model_parallel_rank()
131
+ self.tp_size = get_tensor_model_parallel_world_size()
132
+ self.num_total_experts = config.num_local_experts
133
+ self.top_k = config.num_experts_per_tok
134
+ if self.tp_size > self.num_total_experts:
135
+ raise ValueError(
136
+ f"Tensor parallel size {self.tp_size} is greater than "
137
+ f"the number of experts {self.num_total_experts}.")
138
+ # Split experts equally between ranks
139
+ self.expert_indicies = np.array_split(range(
140
+ self.num_total_experts), self.tp_size)[self.rank].tolist()
141
+ if not self.expert_indicies:
142
+ raise ValueError(
143
+ f"Rank {self.rank} has no experts assigned to it.")
144
+
145
+ self.experts = nn.ModuleList([
146
+ MixtralMLP(self.num_total_experts,
147
+ config.hidden_size,
148
+ config.intermediate_size,
149
+ linear_method=linear_method)
150
+ if idx in self.expert_indicies else DummyModule()
151
+ for idx in range(self.num_total_experts)
152
+ ])
153
+ self.gate = ReplicatedLinear(config.hidden_size,
154
+ self.num_total_experts,
155
+ bias=False,
156
+ linear_method=linear_method)
157
+
158
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
159
+ batch_size, sequence_length, hidden_dim = hidden_states.shape
160
+ hidden_states = hidden_states.view(-1, hidden_dim)
161
+ # router_logits: (batch * sequence_length, n_experts)
162
+ router_logits, _ = self.gate(hidden_states)
163
+
164
+ routing_weights = F.softmax(router_logits, dim=1, dtype=torch.float)
165
+ routing_weights, selected_experts = torch.topk(routing_weights,
166
+ self.top_k,
167
+ dim=-1)
168
+ routing_weights /= routing_weights.sum(dim=-1, keepdim=True)
169
+
170
+ final_hidden_states = None
171
+ for expert_idx in self.expert_indicies:
172
+ expert_layer = self.experts[expert_idx]
173
+ expert_mask = (selected_experts == expert_idx)
174
+ expert_weights = (routing_weights * expert_mask).sum(dim=-1,
175
+ keepdim=True)
176
+
177
+ current_hidden_states = expert_layer(hidden_states).mul_(
178
+ expert_weights)
179
+ if final_hidden_states is None:
180
+ final_hidden_states = current_hidden_states
181
+ else:
182
+ final_hidden_states.add_(current_hidden_states)
183
+
184
+ return tensor_model_parallel_all_reduce(final_hidden_states).view(
185
+ batch_size, sequence_length, hidden_dim)
186
187
188
class MixtralAttention(nn.Module):
@@ -78,6 +193,7 @@ class MixtralAttention(nn.Module):
193
num_kv_heads: int,
194
max_position: int = 4096 * 32,
195
rope_theta: float = 10000,
196
+ linear_method: Optional[LinearMethodBase] = None,
197
sliding_window: Optional[int] = None) -> None:
198
super().__init__()
199
self.hidden_size = hidden_size
@@ -102,24 +218,26 @@ class MixtralAttention(nn.Module):
218
self.rope_theta = rope_theta
219
self.sliding_window = sliding_window
220
105
- self.wqkv = QKVParallelLinear(
221
+ self.qkv_proj = QKVParallelLinear(
222
hidden_size,
223
self.head_dim,
224
self.total_num_heads,
225
self.total_num_kv_heads,
226
bias=False,
227
+ linear_method=linear_method,
228
)
112
- self.wo = RowParallelLinear(
229
+ self.o_proj = RowParallelLinear(
230
self.total_num_heads * self.head_dim,
231
hidden_size,
232
bias=False,
233
+ linear_method=linear_method,
234
)
235
self.rotary_emb = get_rope(
236
self.head_dim,
237
rotary_dim=self.head_dim,
238
max_position=max_position,
239
base=int(self.rope_theta),
122
- is_neox_style=False, # weights not in HF format
240
+ is_neox_style=True,
241
)
242
self.attn = PagedAttention(
243
self.num_heads,
@@ -137,310 +255,74 @@ class MixtralAttention(nn.Module):
255
input_metadata: InputMetadata,
256
cache_event: Optional[torch.cuda.Event],
257
) -> torch.Tensor:
140
- qkv, _ = self.wqkv(hidden_states)
258
+ qkv, _ = self.qkv_proj(hidden_states)
259
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
260
q, k = self.rotary_emb(positions, q, k)
261
k_cache, v_cache = kv_cache
262
attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
263
cache_event)
146
- output, _ = self.wo(attn_output)
264
+ output, _ = self.o_proj(attn_output)
265
return output
266
267
150
-class BlockSparseMoE(nn.Module):
151
- """
152
- Built on the paper and library Megablocks as described in
153
- https://arxiv.org/abs/2211.15841. This implementation is
154
- strictly equivalent to standard MoE with full capacity (no
155
- dropped tokens). It's faster since it formulates MoE operations
156
- in terms of block-sparse operations to accomodate imbalanced
157
- assignments of tokens to experts, whereas standard MoE either
158
- (1) drop tokens at the cost of reduced performance or (2) set
159
- capacity factor to number of experts and thus waste computation
160
- and memory on padding.
161
- """
162
-
163
- def __init__(self, hidden_dim: int, ffn_dim: int, num_experts: int,
164
- top_k: int):
165
- super().__init__()
166
- self.hidden_dim = hidden_dim
167
- self.ffn_dim = ffn_dim
168
- self.num_experts = num_experts
169
- self.top_k = top_k
170
-
171
- # gating
172
- self.gate = nn.Linear(self.hidden_dim,
173
- self.num_experts,
174
- bias=False,
175
- device=torch.cuda.current_device())
176
-
177
- tp_size = get_tensor_model_parallel_world_size()
178
- assert self.ffn_dim % tp_size == 0
179
- self.ffn_dim_per_partition = self.ffn_dim // tp_size
180
- # merged expert weights, all of size (ffn_dim * n_experts, model_dim)
181
- self.w1 = nn.Parameter(
182
- torch.empty(self.ffn_dim_per_partition * self.num_experts,
183
- self.hidden_dim,
184
- device=torch.cuda.current_device()))
185
- set_weight_attrs(self.w1, {"weight_loader": self.moe_weight_loader})
186
- self.w2 = nn.Parameter(
187
- torch.empty(self.ffn_dim_per_partition * self.num_experts,
188
- self.hidden_dim,
189
- device=torch.cuda.current_device()))
190
- set_weight_attrs(self.w2, {"weight_loader": self.moe_weight_loader})
191
- self.w3 = nn.Parameter(
192
- torch.empty(self.ffn_dim_per_partition * self.num_experts,
193
- self.hidden_dim,
194
- device=torch.cuda.current_device()))
195
- set_weight_attrs(self.w3, {"weight_loader": self.moe_weight_loader})
196
-
197
- # Calculate the number of bits needed to represent the expert indices
198
- # so that we can pass it to radix sort.
199
- self.sort_end_bit = max(int(np.ceil(np.log2(self.num_experts))), 1)
200
- self.blocking = 128
201
- self.quantize_scatter_num_bits = -1
202
-
203
- # Calculate the number of bits needed to represent the column indices
204
- # in the intermediate sparse matrix.
205
- max_column_index = (self.ffn_dim * self.num_experts) // self.blocking
206
- self.transpose_sort_end_bit = max(
207
- int(np.ceil(np.log2(max_column_index))), 1)
208
-
209
- def moe_weight_loader(self, param: nn.Parameter,
210
- loaded_weight: torch.Tensor) -> None:
211
- """
212
- Load the weights for the MoE linear layer.
213
- """
214
- tp_rank = get_tensor_model_parallel_rank()
215
- shard_size = self.ffn_dim_per_partition
216
- loaded_weight = loaded_weight.view(self.num_experts, self.ffn_dim, -1)
217
- loaded_weight = loaded_weight[:, shard_size * tp_rank:shard_size *
218
- (tp_rank + 1)]
219
- loaded_weight = loaded_weight.reshape_as(param)
220
- param.data.copy_(loaded_weight)
221
-
222
- def sparse_transpose(
223
- self, size: int, row_indices,
224
- column_indices) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
225
- block_columns = size[1] // self.blocking
226
-
227
- # Sort row indices by column indices to get the transposed matrix's
228
- # column indices.
229
- #
230
- # NOTE: Our sort operation uses the same width indices as the input
231
- # values. To avoid overflow when we have large activation matrices
232
- # we cast to 32-bit before sorting.
233
- _, gather_indices = ops.sort(column_indices.int(),
234
- self.transpose_sort_end_bit)
235
-
236
- # There are a constant number of blocks in every row of the sparse
237
- # matrix. A blocks offset is:
238
- #
239
- # row_index * blocks_per_row + column_index % blocks_per_row
240
- #
241
- # Once we have the block offsets ordered for transposition we can
242
- # divide by blocks_per_row to get the transposed column indices.
243
- column_indices_t = row_indices.gather(0, gather_indices.long())
244
- block_offsets_t = gather_indices.int()
245
-
246
- zero = torch.zeros((1, ), dtype=torch.int32, device=row_indices.device)
247
- nnz_per_column = ops.histogram(column_indices, block_columns)
248
- nnz_per_column = ops.inclusive_cumsum(nnz_per_column, 0)
249
- offsets_t = torch.cat([zero, nnz_per_column])
250
- return column_indices_t, offsets_t, block_offsets_t
251
-
252
- def topology(self, x: torch.Tensor,
253
- padded_bins: torch.Tensor) -> "stk.Matrix":
254
- padded_tokens, _ = x.size()
255
- assert padded_tokens % self.blocking == 0
256
- assert self.ffn_dim_per_partition % self.blocking == 0
257
-
258
- # Offsets for the sparse matrix. All rows have the
259
- # same number of nonzero blocks dictated by the
260
- # dimensionality of a single expert.
261
- block_rows = padded_tokens // self.blocking
262
- blocks_per_row = self.ffn_dim_per_partition // self.blocking
263
- offsets = torch.arange(
264
- 0,
265
- block_rows * blocks_per_row + 1,
266
- blocks_per_row,
267
- dtype=torch.int32,
268
- device=x.device,
269
- )
270
-
271
- # Indices for the sparse matrix. The indices for
272
- # the intermediate matrix are dynamic depending
273
- # on the mapping of tokens to experts.
274
- column_indices = ops.topology(padded_bins, self.blocking, block_rows,
275
- blocks_per_row)
276
-
277
- # TODO(tgale): This is unused. Remove the need for this in stk.
278
- # For now, use meta init to save the device memory.
279
- data = torch.empty(
280
- column_indices.numel(),
281
- self.blocking,
282
- self.blocking,
283
- dtype=x.dtype,
284
- device="meta",
285
- )
286
- shape = (padded_tokens, self.ffn_dim_per_partition * self.num_experts)
287
- row_indices = stk.ops.row_indices(shape, data, offsets, column_indices)
288
- column_indices_t, offsets_t, block_offsets_t = self.sparse_transpose(
289
- shape, row_indices, column_indices)
290
- return stk.Matrix(
291
- shape,
292
- data,
293
- row_indices,
294
- column_indices,
295
- offsets,
296
- column_indices_t,
297
- offsets_t,
298
- block_offsets_t,
299
- )
300
-
301
- def indices_and_padded_bins(
302
- self, selected_experts: torch.Tensor
303
- ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor,
304
- torch.Tensor]:
305
- # Sort the expert ids to produce the scatter/gather
306
- # indices for the permutation.
307
- selected_experts = selected_experts.int()
308
- bin_ids, indices = ops.sort(selected_experts, self.sort_end_bit)
309
-
310
- # Histogram the expert ids to identify the number of
311
- # tokens routed to each expert.
312
- tokens_per_expert = ops.histogram(selected_experts, self.num_experts)
313
-
314
- # Round the token counts up to the block size used in
315
- # the matrix muliplications. Caculate the starting
316
- # position of each bin.
317
- padded_tokens_per_expert = ops.round_up(tokens_per_expert,
318
- self.blocking)
319
- padded_bins = ops.inclusive_cumsum(padded_tokens_per_expert, 0)
320
- padded_bins = promote_scalar(padded_bins)
321
-
322
- # Calculate the bin bounds for the sorted tokens.
323
- bins = ops.inclusive_cumsum(tokens_per_expert, 0)
324
- bins = promote_scalar(bins)
325
- return indices, bin_ids, bins, padded_bins, tokens_per_expert
326
-
327
- @torch.inference_mode()
328
- def forward(self, x: torch.Tensor) -> torch.Tensor:
329
- """
330
- x: (sequence_length, model_dim)
331
- gate_logits: (sequence_length, n_experts)
332
- """
333
- # optional reshape
334
- input_shape = x.shape
335
- x = x.view(-1, input_shape[-1])
336
-
337
- # gate_logits: (sequence_length, n_experts)
338
- gate_logits = self.gate(x)
339
- # all_probs: (sequence_length, n_experts) and upcast for softmax
340
- all_probs = F.softmax(gate_logits, dim=1, dtype=torch.float)
341
- # weights, selected_experts: (sequence_length, top-k)
342
- weights, selected_experts = torch.topk(all_probs, self.top_k, dim=-1)
343
- weights /= weights.sum(dim=-1, keepdim=True)
344
- weights = weights.flatten().to(x.dtype)
345
- selected_experts = selected_experts.flatten()
346
-
347
- (indices, bin_ids, bins, padded_bins,
348
- _) = self.indices_and_padded_bins(selected_experts)
349
-
350
- # Permute tokens and pad to prepare expert computation
351
- # (top_k * sequence_length + padding, model_dim)
352
- x = ops.padded_gather(x, indices, bin_ids, bins, padded_bins,
353
- self.top_k)
354
-
355
- # Create the sparse matrix topology
356
- with torch.no_grad():
357
- topo = self.topology(x, padded_bins)
358
-
359
- # Perform the expert computation
360
- # First Dense x Dense -> Sparse for w1 and w3,
361
- # (top_k * sequence_length + padding, ffn_dim * n_experts)
362
- x = stk.Matrix(
363
- topo.size(),
364
- F.silu(stk.ops.sdd(x, self.w1.t(), topo).data) *
365
- stk.ops.sdd(x, self.w3.t(), topo).data,
366
- topo.row_indices,
367
- topo.column_indices,
368
- topo.offsets,
369
- topo.column_indices_t,
370
- topo.offsets_t,
371
- topo.block_offsets_t,
372
- )
373
-
374
- # Then Sparse x Dense -> Dense for w2
375
- # (top_k * sequence_length + padding, model_dim)
376
- x = stk.ops.dsd(x, self.w2)
377
-
378
- x = tensor_model_parallel_all_reduce(x)
379
-
380
- # Permute back and remove padding
381
- # (top_k * sequence_length, model_dim)
382
- x = ops.padded_scatter(
383
- x,
384
- indices,
385
- bin_ids,
386
- weights,
387
- bins,
388
- padded_bins,
389
- self.top_k,
390
- self.quantize_scatter_num_bits,
391
- )
392
- return x.view(*input_shape)
393
-
394
-
268
class MixtralDecoderLayer(nn.Module):
269
270
def __init__(
271
self,
272
config: MixtralConfig,
273
+ linear_method: Optional[LinearMethodBase] = None,
274
) -> None:
275
super().__init__()
276
self.hidden_size = config.hidden_size
277
# Requires transformers > 4.32.0
278
rope_theta = getattr(config, "rope_theta", 10000)
405
- self.attention = MixtralAttention(
279
+ self.self_attn = MixtralAttention(
280
hidden_size=self.hidden_size,
281
num_heads=config.num_attention_heads,
282
max_position=config.max_position_embeddings,
283
num_kv_heads=config.num_key_value_heads,
284
rope_theta=rope_theta,
411
- sliding_window=config.sliding_window)
412
- self.block_sparse_moe = BlockSparseMoE(
413
- hidden_dim=self.hidden_size,
414
- ffn_dim=config.intermediate_size,
415
- num_experts=config.num_local_experts,
416
- top_k=config.num_experts_per_tok,
417
- )
418
- self.attention_norm = RMSNorm(config.hidden_size,
419
- eps=config.rms_norm_eps)
420
- self.ffn_norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
285
+ sliding_window=config.sliding_window,
286
+ linear_method=linear_method)
287
+ self.block_sparse_moe = MixtralMoE(config=config,
288
+ linear_method=linear_method)
289
+ self.input_layernorm = RMSNorm(config.hidden_size,
290
+ eps=config.rms_norm_eps)
291
+ self.post_attention_layernorm = RMSNorm(config.hidden_size,
292
+ eps=config.rms_norm_eps)
293
294
def forward(
295
self,
296
positions: torch.Tensor,
425
- x: torch.Tensor,
297
+ hidden_states: torch.Tensor,
298
kv_cache: KVCache,
299
input_metadata: InputMetadata,
300
cache_event: Optional[torch.cuda.Event],
301
+ residual: Optional[torch.Tensor],
302
) -> torch.Tensor:
430
- r = self.attention(
303
+ # Self Attention
304
+ if residual is None:
305
+ residual = hidden_states
306
+ hidden_states = self.input_layernorm(hidden_states)
307
+ else:
308
+ hidden_states, residual = self.input_layernorm(
309
+ hidden_states, residual)
310
+ hidden_states = self.self_attn(
311
positions=positions,
432
- hidden_states=self.attention_norm(x),
312
+ hidden_states=hidden_states,
313
kv_cache=kv_cache,
314
input_metadata=input_metadata,
315
cache_event=cache_event,
316
)
437
- h = x + r
438
- r = self.block_sparse_moe(self.ffn_norm(h))
439
- out = h + r
440
- return out
317
318
+ # Fully Connected
319
+ hidden_states, residual = self.post_attention_layernorm(
320
+ hidden_states, residual)
321
+ hidden_states = self.block_sparse_moe(hidden_states)
322
+ return hidden_states, residual
323
443
-class MixtralForCausalLM(nn.Module):
324
+
325
+class MixtralModel(nn.Module):
326
327
def __init__(
328
self,
@@ -448,23 +330,18 @@ class MixtralForCausalLM(nn.Module):
330
linear_method: Optional[LinearMethodBase] = None,
331
) -> None:
332
super().__init__()
451
- self.config = config
452
- assert linear_method is None
333
self.padding_idx = config.pad_token_id
334
self.vocab_size = config.vocab_size
455
- self.tok_embeddings = VocabParallelEmbedding(
335
+
336
+ self.embed_tokens = VocabParallelEmbedding(
337
config.vocab_size,
338
config.hidden_size,
339
)
459
-
460
- self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
461
- self.output = ParallelLMHead(config.vocab_size, config.hidden_size)
462
- self.sampler = Sampler(config.vocab_size)
463
-
340
self.layers = nn.ModuleList([
465
- MixtralDecoderLayer(config)
341
+ MixtralDecoderLayer(config, linear_method=linear_method)
342
for _ in range(config.num_hidden_layers)
343
])
344
+ self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
345
346
def forward(
347
self,
@@ -474,20 +351,42 @@ class MixtralForCausalLM(nn.Module):
351
input_metadata: InputMetadata,
352
cache_events: Optional[List[torch.cuda.Event]],
353
) -> SamplerOutput:
477
- hidden_states = self.tok_embeddings(input_ids)
478
-
479
- # forward
354
+ hidden_states = self.embed_tokens(input_ids)
355
+ residual = None
356
for i in range(len(self.layers)):
357
cache_event = None if cache_events is None else cache_events[i]
358
layer = self.layers[i]
483
- hidden_states = layer(
484
- positions,
485
- hidden_states,
486
- kv_caches[i],
487
- input_metadata,
488
- cache_event,
489
- )
490
- hidden_states = self.norm(hidden_states)
359
+ hidden_states, residual = layer(positions, hidden_states,
360
+ kv_caches[i], input_metadata,
361
+ cache_event, residual)
362
+ hidden_states, _ = self.norm(hidden_states, residual)
363
+ return hidden_states
364
+
365
+
366
+class MixtralForCausalLM(nn.Module):
367
+
368
+ def __init__(
369
+ self,
370
+ config: MixtralConfig,
371
+ linear_method: Optional[LinearMethodBase] = None,
372
+ ) -> None:
373
+ super().__init__()
374
+ self.config = config
375
+ self.linear_method = linear_method
376
+ self.model = MixtralModel(config, linear_method)
377
+ self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
378
+ self.sampler = Sampler(config.vocab_size)
379
+
380
+ def forward(
381
+ self,
382
+ input_ids: torch.Tensor,
383
+ positions: torch.Tensor,
384
+ kv_caches: List[KVCache],
385
+ input_metadata: InputMetadata,
386
+ cache_events: Optional[List[torch.cuda.Event]],
387
+ ) -> torch.Tensor:
388
+ hidden_states = self.model(input_ids, positions, kv_caches,
389
+ input_metadata, cache_events)
390
return hidden_states
391
392
def sample(
@@ -495,7 +394,7 @@ class MixtralForCausalLM(nn.Module):
394
hidden_states: Optional[torch.Tensor],
395
sampling_metadata: SamplingMetadata,
396
) -> SamplerOutput:
498
- next_tokens = self.sampler(self.output.weight, hidden_states,
397
+ next_tokens = self.sampler(self.lm_head.weight, hidden_states,
398
sampling_metadata)
399
return next_tokens
400
@@ -506,10 +405,11 @@ class MixtralForCausalLM(nn.Module):
405
revision: Optional[str] = None):
406
stacked_params_mapping = [
407
# (param_name, shard_name, shard_id)
509
- ("wqkv", "wq", "q"),
510
- ("wqkv", "wk", "k"),
511
- ("wqkv", "wv", "v"),
408
+ ("qkv_proj", "q_proj", "q"),
409
+ ("qkv_proj", "k_proj", "k"),
410
+ ("qkv_proj", "v_proj", "v"),
411
]
412
+
413
params_dict = dict(self.named_parameters())
414
for name, loaded_weight in hf_model_weights_iterator(
415
model_name_or_path, cache_dir, load_format, revision):