@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 21d93c140d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Optimize Mixtral with expert parallelism (#2090)

Antoni Baum committed Dec 13, 2023 at 23:55 UTC 21d93c140d0a97af5f0c59e660cf04bd417fd424
6 files changed +221 -334
Dockerfile
+1 -13
@@ -41,14 +41,6 @@ ENV NVCC_THREADS=$nvcc_threads
41
42 RUN python3 setup.py build_ext --inplace
43
44 -# Build the megablocks library as wheel because it doesn't publish pre-built wheels.
45 -# https://github.com/stanford-futuredata/megablocks/commit/5897cd6f254b7b3edf7a708a3a3314ecb54b6f78
46 -RUN apt-get install -y git && \
47 - git clone https://github.com/stanford-futuredata/megablocks.git && \
48 - cd megablocks && \
49 - git checkout 5897cd6f254b7b3edf7a708a3a3314ecb54b6f78 && \
50 - MAX_JOBS=8 NVCC_THREADS=8 python3 setup.py bdist_wheel
51 -
44 # image to run unit testing suite
45 FROM dev AS test
46
@@ -85,12 +77,8 @@ FROM vllm-base AS vllm-openai
77 RUN --mount=type=cache,target=/root/.cache/pip \
78 pip install accelerate
79
88 -COPY vllm vllm
80 COPY --from=build /workspace/vllm/*.so /workspace/vllm/
90 -COPY --from=build /workspace/megablocks/dist/*.whl /tmp/
91 -RUN --mount=type=cache,target=/root/.cache/pip \
92 - pip install /tmp/megablocks-0.5.0-cp310-cp310-linux_x86_64.whl && \
93 - rm /tmp/megablocks-0.5.0-cp310-cp310-linux_x86_64.whl
81 +COPY vllm vllm
82
83 ENTRYPOINT ["python3", "-m", "vllm.entrypoints.openai.api_server"]
84
README.md
-4
@@ -72,10 +72,6 @@ Install vLLM with pip or [from source](https://vllm.readthedocs.io/en/latest/get
72 ```bash
73 pip install vllm
74 ```
75 -**NOTE:** The Mixtral model additionally requires `megablocks` which can be installed with pip or [from source](https://github.com/stanford-futuredata/megablocks):
76 -```bash
77 -pip install megablocks
78 -```
75
76 ## Getting Started
77
docs/source/models/supported_models.rst
+1 -2
@@ -74,8 +74,7 @@ Otherwise, please refer to :ref:`Adding a New Model <adding_a_new_model>` for in
74 Alternatively, you can raise an issue on our `GitHub <https://github.com/vllm-project/vllm/issues>`_ project.
75
76 .. note::
77 - Currently, the ROCm version of vLLM does not support Mixtral.
78 - Additionally, it only supports Mistral for context lengths up to 4096.
77 + Currently, the ROCm version of vLLM supports Mistral and Mixtral only for context lengths up to 4096.
78
79 .. tip::
80 The easiest way to check if your model is supported is to run the program below:
vllm/config.py
+9 -7
@@ -120,14 +120,16 @@ class ModelConfig:
120 if load_format == "auto":
121 load_format = "pt"
122
123 - # FIXME(woosuk): This is a temporary hack. Support safetensor weights.
123 + # TODO: Remove this check once HF updates the pt weights of Mixtral.
124 architectures = getattr(self.hf_config, "architectures", [])
125 - if "MixtralForCausalLM" in architectures and load_format != "pt":
126 - logger.info(
127 - "Currently, only 'pt' format is supported for Mixtral. "
128 - "Changing the format to 'pt'. This may re-download the "
129 - "weights if you have downloaded the safetensor weights.")
130 - load_format = "pt"
125 + if "MixtralForCausalLM" in architectures:
126 + if load_format == "pt":
127 + raise ValueError(
128 + "Currently, the 'pt' format is not supported for Mixtral. "
129 + "Please use the 'safetensors' format instead. ")
130 + elif load_format == "auto":
131 + # Do not fall back to pt weights.
132 + load_format = "safetensors"
133
134 self.load_format = load_format
135
vllm/model_executor/models/__init__.py
+3 -1
@@ -39,13 +39,15 @@ _MODELS = {
39 }
40
41 # Models not supported by ROCm.
42 -_ROCM_UNSUPPORTED_MODELS = ["MixtralForCausalLM"]
42 +_ROCM_UNSUPPORTED_MODELS = []
43
44 # Models partially supported by ROCm.
45 # Architecture -> Reason.
46 _ROCM_PARTIALLY_SUPPORTED_MODELS = {
47 "MistralForCausalLM":
48 "Sliding window attention is not yet supported in ROCm's flash attention",
49 + "MixtralForCausalLM":
50 + "Sliding window attention is not yet supported in ROCm's flash attention",
51 }
52
53
vllm/model_executor/models/mixtral.py
+207 -307
@@ -31,22 +31,11 @@ import torch.nn.functional as F
31 from torch import nn
32 from transformers import MixtralConfig
33
34 -try:
35 - import megablocks.ops as ops
36 -except ImportError as e:
37 - raise ImportError("MegaBlocks not found. "
38 - "Please install it by `pip install megablocks`.") from e
39 -try:
40 - import stk
41 -except ImportError as e:
42 - raise ImportError(
43 - "STK not found. "
44 - "Please install it by `pip install stanford-stk`.") from e
45 -
34 from vllm.model_executor.input_metadata import InputMetadata
35 from vllm.model_executor.layers.attention import PagedAttention
36 from vllm.model_executor.layers.layernorm import RMSNorm
37 from vllm.model_executor.layers.linear import (LinearMethodBase,
38 + ReplicatedLinear,
39 QKVParallelLinear,
40 RowParallelLinear)
41 from vllm.model_executor.layers.rotary_embedding import get_rope
@@ -66,8 +55,134 @@ from vllm.sequence import SamplerOutput
55 KVCache = Tuple[torch.Tensor, torch.Tensor]
56
57
69 -def promote_scalar(x: torch.Tensor) -> torch.Tensor:
70 - return x.view(1) if len(x.size()) == 0 else x
58 +class MixtralMLP(nn.Module):
59 +
60 + def __init__(
61 + self,
62 + num_experts: int,
63 + hidden_size: int,
64 + intermediate_size: int,
65 + linear_method: Optional[LinearMethodBase] = None,
66 + ) -> None:
67 + super().__init__()
68 + self.num_experts = num_experts
69 + self.ffn_dim = intermediate_size
70 + self.hidden_dim = hidden_size
71 +
72 + self.w1 = ReplicatedLinear(self.hidden_dim,
73 + self.ffn_dim,
74 + bias=False,
75 + linear_method=linear_method)
76 + self.w2 = ReplicatedLinear(self.ffn_dim,
77 + self.hidden_dim,
78 + bias=False,
79 + linear_method=linear_method)
80 + self.w3 = ReplicatedLinear(self.hidden_dim,
81 + self.ffn_dim,
82 + bias=False,
83 + linear_method=linear_method)
84 +
85 + # TODO: Use vllm's SiluAndMul
86 + self.act_fn = nn.SiLU()
87 +
88 + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
89 + w1_out, _ = self.w1(hidden_states)
90 + w1_out = self.act_fn(w1_out)
91 + w3_out, _ = self.w3(hidden_states)
92 + current_hidden_states = w1_out * w3_out
93 + current_hidden_states, _ = self.w2(current_hidden_states)
94 + return current_hidden_states
95 +
96 +
97 +class DummyModule(nn.Module):
98 +
99 + def __init__(self) -> None:
100 + super().__init__()
101 +
102 + self.w1 = nn.Linear(0, 0, bias=False)
103 + self.w2 = nn.Linear(0, 0, bias=False)
104 + self.w3 = nn.Linear(0, 0, bias=False)
105 +
106 + set_weight_attrs(self.w1.weight,
107 + {"weight_loader": self.dummy_weight_loader})
108 + set_weight_attrs(self.w2.weight,
109 + {"weight_loader": self.dummy_weight_loader})
110 + set_weight_attrs(self.w3.weight,
111 + {"weight_loader": self.dummy_weight_loader})
112 +
113 + def forward(self, *args, **kwargs) -> None:
114 + raise NotImplementedError()
115 +
116 + def dummy_weight_loader(self, *args, **kwargs) -> None: # pylint: disable=unused-argument
117 + # Noop
118 + return
119 +
120 +
121 +class MixtralMoE(nn.Module):
122 +
123 + def __init__(
124 + self,
125 + config: MixtralConfig,
126 + linear_method: Optional[LinearMethodBase] = None,
127 + ):
128 + super().__init__()
129 + self.config = config
130 + self.rank = get_tensor_model_parallel_rank()
131 + self.tp_size = get_tensor_model_parallel_world_size()
132 + self.num_total_experts = config.num_local_experts
133 + self.top_k = config.num_experts_per_tok
134 + if self.tp_size > self.num_total_experts:
135 + raise ValueError(
136 + f"Tensor parallel size {self.tp_size} is greater than "
137 + f"the number of experts {self.num_total_experts}.")
138 + # Split experts equally between ranks
139 + self.expert_indicies = np.array_split(range(
140 + self.num_total_experts), self.tp_size)[self.rank].tolist()
141 + if not self.expert_indicies:
142 + raise ValueError(
143 + f"Rank {self.rank} has no experts assigned to it.")
144 +
145 + self.experts = nn.ModuleList([
146 + MixtralMLP(self.num_total_experts,
147 + config.hidden_size,
148 + config.intermediate_size,
149 + linear_method=linear_method)
150 + if idx in self.expert_indicies else DummyModule()
151 + for idx in range(self.num_total_experts)
152 + ])
153 + self.gate = ReplicatedLinear(config.hidden_size,
154 + self.num_total_experts,
155 + bias=False,
156 + linear_method=linear_method)
157 +
158 + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
159 + batch_size, sequence_length, hidden_dim = hidden_states.shape
160 + hidden_states = hidden_states.view(-1, hidden_dim)
161 + # router_logits: (batch * sequence_length, n_experts)
162 + router_logits, _ = self.gate(hidden_states)
163 +
164 + routing_weights = F.softmax(router_logits, dim=1, dtype=torch.float)
165 + routing_weights, selected_experts = torch.topk(routing_weights,
166 + self.top_k,
167 + dim=-1)
168 + routing_weights /= routing_weights.sum(dim=-1, keepdim=True)
169 +
170 + final_hidden_states = None
171 + for expert_idx in self.expert_indicies:
172 + expert_layer = self.experts[expert_idx]
173 + expert_mask = (selected_experts == expert_idx)
174 + expert_weights = (routing_weights * expert_mask).sum(dim=-1,
175 + keepdim=True)
176 +
177 + current_hidden_states = expert_layer(hidden_states).mul_(
178 + expert_weights)
179 + if final_hidden_states is None:
180 + final_hidden_states = current_hidden_states
181 + else:
182 + final_hidden_states.add_(current_hidden_states)
183 +
184 + return tensor_model_parallel_all_reduce(final_hidden_states).view(
185 + batch_size, sequence_length, hidden_dim)
186
187
188 class MixtralAttention(nn.Module):
@@ -78,6 +193,7 @@ class MixtralAttention(nn.Module):
193 num_kv_heads: int,
194 max_position: int = 4096 * 32,
195 rope_theta: float = 10000,
196 + linear_method: Optional[LinearMethodBase] = None,
197 sliding_window: Optional[int] = None) -> None:
198 super().__init__()
199 self.hidden_size = hidden_size
@@ -102,24 +218,26 @@ class MixtralAttention(nn.Module):
218 self.rope_theta = rope_theta
219 self.sliding_window = sliding_window
220
105 - self.wqkv = QKVParallelLinear(
221 + self.qkv_proj = QKVParallelLinear(
222 hidden_size,
223 self.head_dim,
224 self.total_num_heads,
225 self.total_num_kv_heads,
226 bias=False,
227 + linear_method=linear_method,
228 )
112 - self.wo = RowParallelLinear(
229 + self.o_proj = RowParallelLinear(
230 self.total_num_heads * self.head_dim,
231 hidden_size,
232 bias=False,
233 + linear_method=linear_method,
234 )
235 self.rotary_emb = get_rope(
236 self.head_dim,
237 rotary_dim=self.head_dim,
238 max_position=max_position,
239 base=int(self.rope_theta),
122 - is_neox_style=False, # weights not in HF format
240 + is_neox_style=True,
241 )
242 self.attn = PagedAttention(
243 self.num_heads,
@@ -137,310 +255,74 @@ class MixtralAttention(nn.Module):
255 input_metadata: InputMetadata,
256 cache_event: Optional[torch.cuda.Event],
257 ) -> torch.Tensor:
140 - qkv, _ = self.wqkv(hidden_states)
258 + qkv, _ = self.qkv_proj(hidden_states)
259 q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
260 q, k = self.rotary_emb(positions, q, k)
261 k_cache, v_cache = kv_cache
262 attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
263 cache_event)
146 - output, _ = self.wo(attn_output)
264 + output, _ = self.o_proj(attn_output)
265 return output
266
267
150 -class BlockSparseMoE(nn.Module):
151 - """
152 - Built on the paper and library Megablocks as described in
153 - https://arxiv.org/abs/2211.15841. This implementation is
154 - strictly equivalent to standard MoE with full capacity (no
155 - dropped tokens). It's faster since it formulates MoE operations
156 - in terms of block-sparse operations to accomodate imbalanced
157 - assignments of tokens to experts, whereas standard MoE either
158 - (1) drop tokens at the cost of reduced performance or (2) set
159 - capacity factor to number of experts and thus waste computation
160 - and memory on padding.
161 - """
162 -
163 - def __init__(self, hidden_dim: int, ffn_dim: int, num_experts: int,
164 - top_k: int):
165 - super().__init__()
166 - self.hidden_dim = hidden_dim
167 - self.ffn_dim = ffn_dim
168 - self.num_experts = num_experts
169 - self.top_k = top_k
170 -
171 - # gating
172 - self.gate = nn.Linear(self.hidden_dim,
173 - self.num_experts,
174 - bias=False,
175 - device=torch.cuda.current_device())
176 -
177 - tp_size = get_tensor_model_parallel_world_size()
178 - assert self.ffn_dim % tp_size == 0
179 - self.ffn_dim_per_partition = self.ffn_dim // tp_size
180 - # merged expert weights, all of size (ffn_dim * n_experts, model_dim)
181 - self.w1 = nn.Parameter(
182 - torch.empty(self.ffn_dim_per_partition * self.num_experts,
183 - self.hidden_dim,
184 - device=torch.cuda.current_device()))
185 - set_weight_attrs(self.w1, {"weight_loader": self.moe_weight_loader})
186 - self.w2 = nn.Parameter(
187 - torch.empty(self.ffn_dim_per_partition * self.num_experts,
188 - self.hidden_dim,
189 - device=torch.cuda.current_device()))
190 - set_weight_attrs(self.w2, {"weight_loader": self.moe_weight_loader})
191 - self.w3 = nn.Parameter(
192 - torch.empty(self.ffn_dim_per_partition * self.num_experts,
193 - self.hidden_dim,
194 - device=torch.cuda.current_device()))
195 - set_weight_attrs(self.w3, {"weight_loader": self.moe_weight_loader})
196 -
197 - # Calculate the number of bits needed to represent the expert indices
198 - # so that we can pass it to radix sort.
199 - self.sort_end_bit = max(int(np.ceil(np.log2(self.num_experts))), 1)
200 - self.blocking = 128
201 - self.quantize_scatter_num_bits = -1
202 -
203 - # Calculate the number of bits needed to represent the column indices
204 - # in the intermediate sparse matrix.
205 - max_column_index = (self.ffn_dim * self.num_experts) // self.blocking
206 - self.transpose_sort_end_bit = max(
207 - int(np.ceil(np.log2(max_column_index))), 1)
208 -
209 - def moe_weight_loader(self, param: nn.Parameter,
210 - loaded_weight: torch.Tensor) -> None:
211 - """
212 - Load the weights for the MoE linear layer.
213 - """
214 - tp_rank = get_tensor_model_parallel_rank()
215 - shard_size = self.ffn_dim_per_partition
216 - loaded_weight = loaded_weight.view(self.num_experts, self.ffn_dim, -1)
217 - loaded_weight = loaded_weight[:, shard_size * tp_rank:shard_size *
218 - (tp_rank + 1)]
219 - loaded_weight = loaded_weight.reshape_as(param)
220 - param.data.copy_(loaded_weight)
221 -
222 - def sparse_transpose(
223 - self, size: int, row_indices,
224 - column_indices) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
225 - block_columns = size[1] // self.blocking
226 -
227 - # Sort row indices by column indices to get the transposed matrix's
228 - # column indices.
229 - #
230 - # NOTE: Our sort operation uses the same width indices as the input
231 - # values. To avoid overflow when we have large activation matrices
232 - # we cast to 32-bit before sorting.
233 - _, gather_indices = ops.sort(column_indices.int(),
234 - self.transpose_sort_end_bit)
235 -
236 - # There are a constant number of blocks in every row of the sparse
237 - # matrix. A blocks offset is:
238 - #
239 - # row_index * blocks_per_row + column_index % blocks_per_row
240 - #
241 - # Once we have the block offsets ordered for transposition we can
242 - # divide by blocks_per_row to get the transposed column indices.
243 - column_indices_t = row_indices.gather(0, gather_indices.long())
244 - block_offsets_t = gather_indices.int()
245 -
246 - zero = torch.zeros((1, ), dtype=torch.int32, device=row_indices.device)
247 - nnz_per_column = ops.histogram(column_indices, block_columns)
248 - nnz_per_column = ops.inclusive_cumsum(nnz_per_column, 0)
249 - offsets_t = torch.cat([zero, nnz_per_column])
250 - return column_indices_t, offsets_t, block_offsets_t
251 -
252 - def topology(self, x: torch.Tensor,
253 - padded_bins: torch.Tensor) -> "stk.Matrix":
254 - padded_tokens, _ = x.size()
255 - assert padded_tokens % self.blocking == 0
256 - assert self.ffn_dim_per_partition % self.blocking == 0
257 -
258 - # Offsets for the sparse matrix. All rows have the
259 - # same number of nonzero blocks dictated by the
260 - # dimensionality of a single expert.
261 - block_rows = padded_tokens // self.blocking
262 - blocks_per_row = self.ffn_dim_per_partition // self.blocking
263 - offsets = torch.arange(
264 - 0,
265 - block_rows * blocks_per_row + 1,
266 - blocks_per_row,
267 - dtype=torch.int32,
268 - device=x.device,
269 - )
270 -
271 - # Indices for the sparse matrix. The indices for
272 - # the intermediate matrix are dynamic depending
273 - # on the mapping of tokens to experts.
274 - column_indices = ops.topology(padded_bins, self.blocking, block_rows,
275 - blocks_per_row)
276 -
277 - # TODO(tgale): This is unused. Remove the need for this in stk.
278 - # For now, use meta init to save the device memory.
279 - data = torch.empty(
280 - column_indices.numel(),
281 - self.blocking,
282 - self.blocking,
283 - dtype=x.dtype,
284 - device="meta",
285 - )
286 - shape = (padded_tokens, self.ffn_dim_per_partition * self.num_experts)
287 - row_indices = stk.ops.row_indices(shape, data, offsets, column_indices)
288 - column_indices_t, offsets_t, block_offsets_t = self.sparse_transpose(
289 - shape, row_indices, column_indices)
290 - return stk.Matrix(
291 - shape,
292 - data,
293 - row_indices,
294 - column_indices,
295 - offsets,
296 - column_indices_t,
297 - offsets_t,
298 - block_offsets_t,
299 - )
300 -
301 - def indices_and_padded_bins(
302 - self, selected_experts: torch.Tensor
303 - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor,
304 - torch.Tensor]:
305 - # Sort the expert ids to produce the scatter/gather
306 - # indices for the permutation.
307 - selected_experts = selected_experts.int()
308 - bin_ids, indices = ops.sort(selected_experts, self.sort_end_bit)
309 -
310 - # Histogram the expert ids to identify the number of
311 - # tokens routed to each expert.
312 - tokens_per_expert = ops.histogram(selected_experts, self.num_experts)
313 -
314 - # Round the token counts up to the block size used in
315 - # the matrix muliplications. Caculate the starting
316 - # position of each bin.
317 - padded_tokens_per_expert = ops.round_up(tokens_per_expert,
318 - self.blocking)
319 - padded_bins = ops.inclusive_cumsum(padded_tokens_per_expert, 0)
320 - padded_bins = promote_scalar(padded_bins)
321 -
322 - # Calculate the bin bounds for the sorted tokens.
323 - bins = ops.inclusive_cumsum(tokens_per_expert, 0)
324 - bins = promote_scalar(bins)
325 - return indices, bin_ids, bins, padded_bins, tokens_per_expert
326 -
327 - @torch.inference_mode()
328 - def forward(self, x: torch.Tensor) -> torch.Tensor:
329 - """
330 - x: (sequence_length, model_dim)
331 - gate_logits: (sequence_length, n_experts)
332 - """
333 - # optional reshape
334 - input_shape = x.shape
335 - x = x.view(-1, input_shape[-1])
336 -
337 - # gate_logits: (sequence_length, n_experts)
338 - gate_logits = self.gate(x)
339 - # all_probs: (sequence_length, n_experts) and upcast for softmax
340 - all_probs = F.softmax(gate_logits, dim=1, dtype=torch.float)
341 - # weights, selected_experts: (sequence_length, top-k)
342 - weights, selected_experts = torch.topk(all_probs, self.top_k, dim=-1)
343 - weights /= weights.sum(dim=-1, keepdim=True)
344 - weights = weights.flatten().to(x.dtype)
345 - selected_experts = selected_experts.flatten()
346 -
347 - (indices, bin_ids, bins, padded_bins,
348 - _) = self.indices_and_padded_bins(selected_experts)
349 -
350 - # Permute tokens and pad to prepare expert computation
351 - # (top_k * sequence_length + padding, model_dim)
352 - x = ops.padded_gather(x, indices, bin_ids, bins, padded_bins,
353 - self.top_k)
354 -
355 - # Create the sparse matrix topology
356 - with torch.no_grad():
357 - topo = self.topology(x, padded_bins)
358 -
359 - # Perform the expert computation
360 - # First Dense x Dense -> Sparse for w1 and w3,
361 - # (top_k * sequence_length + padding, ffn_dim * n_experts)
362 - x = stk.Matrix(
363 - topo.size(),
364 - F.silu(stk.ops.sdd(x, self.w1.t(), topo).data) *
365 - stk.ops.sdd(x, self.w3.t(), topo).data,
366 - topo.row_indices,
367 - topo.column_indices,
368 - topo.offsets,
369 - topo.column_indices_t,
370 - topo.offsets_t,
371 - topo.block_offsets_t,
372 - )
373 -
374 - # Then Sparse x Dense -> Dense for w2
375 - # (top_k * sequence_length + padding, model_dim)
376 - x = stk.ops.dsd(x, self.w2)
377 -
378 - x = tensor_model_parallel_all_reduce(x)
379 -
380 - # Permute back and remove padding
381 - # (top_k * sequence_length, model_dim)
382 - x = ops.padded_scatter(
383 - x,
384 - indices,
385 - bin_ids,
386 - weights,
387 - bins,
388 - padded_bins,
389 - self.top_k,
390 - self.quantize_scatter_num_bits,
391 - )
392 - return x.view(*input_shape)
393 -
394 -
268 class MixtralDecoderLayer(nn.Module):
269
270 def __init__(
271 self,
272 config: MixtralConfig,
273 + linear_method: Optional[LinearMethodBase] = None,
274 ) -> None:
275 super().__init__()
276 self.hidden_size = config.hidden_size
277 # Requires transformers > 4.32.0
278 rope_theta = getattr(config, "rope_theta", 10000)
405 - self.attention = MixtralAttention(
279 + self.self_attn = MixtralAttention(
280 hidden_size=self.hidden_size,
281 num_heads=config.num_attention_heads,
282 max_position=config.max_position_embeddings,
283 num_kv_heads=config.num_key_value_heads,
284 rope_theta=rope_theta,
411 - sliding_window=config.sliding_window)
412 - self.block_sparse_moe = BlockSparseMoE(
413 - hidden_dim=self.hidden_size,
414 - ffn_dim=config.intermediate_size,
415 - num_experts=config.num_local_experts,
416 - top_k=config.num_experts_per_tok,
417 - )
418 - self.attention_norm = RMSNorm(config.hidden_size,
419 - eps=config.rms_norm_eps)
420 - self.ffn_norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
285 + sliding_window=config.sliding_window,
286 + linear_method=linear_method)
287 + self.block_sparse_moe = MixtralMoE(config=config,
288 + linear_method=linear_method)
289 + self.input_layernorm = RMSNorm(config.hidden_size,
290 + eps=config.rms_norm_eps)
291 + self.post_attention_layernorm = RMSNorm(config.hidden_size,
292 + eps=config.rms_norm_eps)
293
294 def forward(
295 self,
296 positions: torch.Tensor,
425 - x: torch.Tensor,
297 + hidden_states: torch.Tensor,
298 kv_cache: KVCache,
299 input_metadata: InputMetadata,
300 cache_event: Optional[torch.cuda.Event],
301 + residual: Optional[torch.Tensor],
302 ) -> torch.Tensor:
430 - r = self.attention(
303 + # Self Attention
304 + if residual is None:
305 + residual = hidden_states
306 + hidden_states = self.input_layernorm(hidden_states)
307 + else:
308 + hidden_states, residual = self.input_layernorm(
309 + hidden_states, residual)
310 + hidden_states = self.self_attn(
311 positions=positions,
432 - hidden_states=self.attention_norm(x),
312 + hidden_states=hidden_states,
313 kv_cache=kv_cache,
314 input_metadata=input_metadata,
315 cache_event=cache_event,
316 )
437 - h = x + r
438 - r = self.block_sparse_moe(self.ffn_norm(h))
439 - out = h + r
440 - return out
317
318 + # Fully Connected
319 + hidden_states, residual = self.post_attention_layernorm(
320 + hidden_states, residual)
321 + hidden_states = self.block_sparse_moe(hidden_states)
322 + return hidden_states, residual
323
443 -class MixtralForCausalLM(nn.Module):
324 +
325 +class MixtralModel(nn.Module):
326
327 def __init__(
328 self,
@@ -448,23 +330,18 @@ class MixtralForCausalLM(nn.Module):
330 linear_method: Optional[LinearMethodBase] = None,
331 ) -> None:
332 super().__init__()
451 - self.config = config
452 - assert linear_method is None
333 self.padding_idx = config.pad_token_id
334 self.vocab_size = config.vocab_size
455 - self.tok_embeddings = VocabParallelEmbedding(
335 +
336 + self.embed_tokens = VocabParallelEmbedding(
337 config.vocab_size,
338 config.hidden_size,
339 )
459 -
460 - self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
461 - self.output = ParallelLMHead(config.vocab_size, config.hidden_size)
462 - self.sampler = Sampler(config.vocab_size)
463 -
340 self.layers = nn.ModuleList([
465 - MixtralDecoderLayer(config)
341 + MixtralDecoderLayer(config, linear_method=linear_method)
342 for _ in range(config.num_hidden_layers)
343 ])
344 + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
345
346 def forward(
347 self,
@@ -474,20 +351,42 @@ class MixtralForCausalLM(nn.Module):
351 input_metadata: InputMetadata,
352 cache_events: Optional[List[torch.cuda.Event]],
353 ) -> SamplerOutput:
477 - hidden_states = self.tok_embeddings(input_ids)
478 -
479 - # forward
354 + hidden_states = self.embed_tokens(input_ids)
355 + residual = None
356 for i in range(len(self.layers)):
357 cache_event = None if cache_events is None else cache_events[i]
358 layer = self.layers[i]
483 - hidden_states = layer(
484 - positions,
485 - hidden_states,
486 - kv_caches[i],
487 - input_metadata,
488 - cache_event,
489 - )
490 - hidden_states = self.norm(hidden_states)
359 + hidden_states, residual = layer(positions, hidden_states,
360 + kv_caches[i], input_metadata,
361 + cache_event, residual)
362 + hidden_states, _ = self.norm(hidden_states, residual)
363 + return hidden_states
364 +
365 +
366 +class MixtralForCausalLM(nn.Module):
367 +
368 + def __init__(
369 + self,
370 + config: MixtralConfig,
371 + linear_method: Optional[LinearMethodBase] = None,
372 + ) -> None:
373 + super().__init__()
374 + self.config = config
375 + self.linear_method = linear_method
376 + self.model = MixtralModel(config, linear_method)
377 + self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
378 + self.sampler = Sampler(config.vocab_size)
379 +
380 + def forward(
381 + self,
382 + input_ids: torch.Tensor,
383 + positions: torch.Tensor,
384 + kv_caches: List[KVCache],
385 + input_metadata: InputMetadata,
386 + cache_events: Optional[List[torch.cuda.Event]],
387 + ) -> torch.Tensor:
388 + hidden_states = self.model(input_ids, positions, kv_caches,
389 + input_metadata, cache_events)
390 return hidden_states
391
392 def sample(
@@ -495,7 +394,7 @@ class MixtralForCausalLM(nn.Module):
394 hidden_states: Optional[torch.Tensor],
395 sampling_metadata: SamplingMetadata,
396 ) -> SamplerOutput:
498 - next_tokens = self.sampler(self.output.weight, hidden_states,
397 + next_tokens = self.sampler(self.lm_head.weight, hidden_states,
398 sampling_metadata)
399 return next_tokens
400
@@ -506,10 +405,11 @@ class MixtralForCausalLM(nn.Module):
405 revision: Optional[str] = None):
406 stacked_params_mapping = [
407 # (param_name, shard_name, shard_id)
509 - ("wqkv", "wq", "q"),
510 - ("wqkv", "wk", "k"),
511 - ("wqkv", "wv", "v"),
408 + ("qkv_proj", "q_proj", "q"),
409 + ("qkv_proj", "k_proj", "k"),
410 + ("qkv_proj", "v_proj", "v"),
411 ]
412 +
413 params_dict = dict(self.named_parameters())
414 for name, loaded_weight in hf_model_weights_iterator(
415 model_name_or_path, cache_dir, load_format, revision):