A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Model] use FusedMoE layer in Jamba (#6935)
Avshalom Manevich committed
Jul 31, 2024 at 22:00 UTC
2ee8d3ba55f1175162dbc8e70b76674197b127c6
1 file changed
+49
-108
vllm/model_executor/models/jamba.py
+49
-108
@@ -1,5 +1,5 @@
1
# coding=utf-8
2
-"""Inference-only Jurassic model."""
2
+"""Inference-only Jamba model."""
3
from dataclasses import dataclass
4
from typing import Dict, Iterable, List, Optional, Tuple
5
@@ -15,10 +15,9 @@ from vllm.attention.backends.abstract import AttentionMetadata
15
from vllm.attention.layer import Attention
16
from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig
17
from vllm.distributed import (get_tensor_model_parallel_rank,
18
- get_tensor_model_parallel_world_size,
19
- tensor_model_parallel_all_reduce)
18
+ get_tensor_model_parallel_world_size)
19
from vllm.model_executor.layers.activation import SiluAndMul
21
-from vllm.model_executor.layers.fused_moe import fused_moe
20
+from vllm.model_executor.layers.fused_moe import FusedMoE
21
from vllm.model_executor.layers.layernorm import RMSNorm
22
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
23
MergedColumnParallelLinear,
@@ -282,108 +281,50 @@ class JambaMLP(nn.Module):
281
282
283
class JambaMoE(nn.Module):
285
- """A tensor-parallel MoE implementation for Mixtral that shards each expert
286
- across all ranks.
284
288
- Each expert's weights are sharded across all ranks and a fused MoE
289
- kernel is used for the forward pass, and finally we reduce the outputs
290
- across ranks.
291
- """
292
-
293
- def __init__(
294
- self,
295
- config: JambaConfig,
296
- params_dtype: Optional[torch.dtype] = None,
297
- tp_size: Optional[int] = None,
298
- quant_config: Optional[QuantizationConfig] = None,
299
- ):
285
+ def __init__(self,
286
+ config: JambaConfig,
287
+ num_experts: Optional[int] = None,
288
+ top_k: Optional[int] = None,
289
+ params_dtype: Optional[torch.dtype] = None,
290
+ tp_size: Optional[int] = None,
291
+ quant_config: Optional[QuantizationConfig] = None):
292
super().__init__()
301
- self.tp_size = tp_size or get_tensor_model_parallel_world_size()
302
- self.num_total_experts = config.num_experts
303
- self.top_k = config.num_experts_per_tok
293
+ self.num_total_experts = num_experts or config.num_experts
294
+ self.top_k = top_k or config.num_experts_per_tok
295
self.hidden_size = config.hidden_size
305
- self.intermediate_size = config.intermediate_size // self.tp_size
306
-
307
- if params_dtype is None:
308
- params_dtype = torch.get_default_dtype()
309
- self.params_dtype = params_dtype
310
-
311
- self.router = ReplicatedLinear(self.hidden_size,
312
- self.num_total_experts,
313
- bias=False,
314
- params_dtype=self.params_dtype)
315
-
316
- self.ws = nn.Parameter(
317
- torch.empty(
318
- self.num_total_experts,
319
- 2 * self.intermediate_size,
320
- self.hidden_size,
321
- device="cuda",
322
- dtype=self.params_dtype,
323
- ))
324
- self.w2s = nn.Parameter(
325
- torch.empty(
326
- self.num_total_experts,
327
- self.hidden_size,
328
- self.intermediate_size,
329
- device="cuda",
330
- dtype=self.params_dtype,
331
- ))
296
+ self.intermediate_size = config.intermediate_size
297
333
- set_weight_attrs(
334
- self.ws,
335
- {
336
- "weight_loader": self.weight_loader,
337
- },
338
- )
339
- set_weight_attrs(
340
- self.w2s,
341
- {
342
- "weight_loader": self.weight_loader,
343
- },
344
- )
345
-
346
- def weight_loader(
347
- self,
348
- param: nn.Parameter,
349
- loaded_weight: torch.Tensor,
350
- weight_name: str,
351
- expert_id: int,
352
- ):
353
- tp_rank = get_tensor_model_parallel_rank()
354
- param_data = param.data
355
- shard_size = self.intermediate_size
356
- shard = slice(tp_rank * shard_size, (tp_rank + 1) * shard_size)
357
- if weight_name.endswith("gate_proj.weight"):
358
- param_data[expert_id, 0:shard_size, :] = loaded_weight[shard, :]
359
- if weight_name.endswith("up_proj.weight"):
360
- param_data[expert_id,
361
- shard_size:2 * shard_size, :] = loaded_weight[shard, :]
362
- if weight_name.endswith("down_proj.weight"):
363
- param_data[expert_id, :, :] = loaded_weight[:, shard]
298
+ if self.num_total_experts > 1:
299
+ self.router = ReplicatedLinear(self.hidden_size,
300
+ self.num_total_experts,
301
+ bias=False,
302
+ quant_config=None,
303
+ params_dtype=params_dtype)
304
+
305
+ self.experts = FusedMoE(self.num_total_experts,
306
+ self.top_k,
307
+ self.hidden_size,
308
+ self.intermediate_size,
309
+ tp_size=tp_size,
310
+ params_dtype=params_dtype,
311
+ reduce_results=True,
312
+ renormalize=False,
313
+ use_grouped_topk=False,
314
+ quant_config=quant_config)
315
316
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
366
- num_tokens, hidden_size = hidden_states.shape
317
+ orig_shape = hidden_states.shape
318
hidden_states = hidden_states.view(-1, self.hidden_size)
319
# router_logits: (batch * sequence_length, n_experts)
369
- router_logits, _ = self.router(hidden_states)
370
-
371
- final_hidden_states = fused_moe(
372
- hidden_states,
373
- self.ws,
374
- self.w2s,
375
- router_logits,
376
- self.top_k,
377
- renormalize=
378
- False, # Mixtral normalize the expert probs to 1. We don't!
379
- inplace=True,
380
- )
381
-
382
- if self.tp_size > 1:
383
- final_hidden_states = tensor_model_parallel_all_reduce(
384
- final_hidden_states)
385
-
386
- return final_hidden_states.view(num_tokens, hidden_size)
320
+ if self.num_total_experts > 1:
321
+ router_logits, _ = self.router(hidden_states)
322
+ else:
323
+ router_logits = torch.ones((hidden_states.shape[0], 1),
324
+ device=hidden_states.device,
325
+ dtype=hidden_states.dtype)
326
+ hidden_states = self.experts(hidden_states, router_logits)
327
+ return hidden_states.view(orig_shape)
328
329
330
class JambaMambaDecoderLayer(nn.Module):
@@ -917,15 +858,13 @@ class JambaForCausalLM(nn.Module, HasInnerState):
858
("gate_up_proj", "up_proj", 1),
859
]
860
920
- expert_params_mapping = [
921
- # (param_name, weight_name, expert_id)
922
- (
923
- "ws" if weight_name in ["gate_proj", "up_proj"] else "w2s",
924
- f"experts.{expert_id}.{weight_name}.weight",
925
- expert_id,
926
- ) for expert_id in range(self.config.num_experts)
927
- for weight_name in ["down_proj", "up_proj", "gate_proj"]
928
- ]
861
+ # Params for weights, fp8 weight scales, fp8 activation scales
862
+ # (param_name, weight_name, expert_id, shard_id)
863
+ expert_params_mapping = FusedMoE.make_expert_params_mapping(
864
+ ckpt_gate_proj_name="gate_proj",
865
+ ckpt_down_proj_name="down_proj",
866
+ ckpt_up_proj_name="up_proj",
867
+ num_experts=self.config.num_experts)
868
869
params_dict = dict(self.named_parameters())
870
for name, loaded_weight in weights:
@@ -952,7 +891,8 @@ class JambaForCausalLM(nn.Module, HasInnerState):
891
weight_loader(param, loaded_weight, shard_id)
892
break
893
else:
955
- for param_name, weight_name, expert_id in expert_params_mapping:
894
+ for mapping in expert_params_mapping:
895
+ param_name, weight_name, expert_id, shard_id = mapping
896
if weight_name not in name:
897
continue
898
name = name.replace(weight_name, param_name)
@@ -961,6 +901,7 @@ class JambaForCausalLM(nn.Module, HasInnerState):
901
weight_loader(param,
902
loaded_weight,
903
weight_name,
904
+ shard_id=shard_id,
905
expert_id=expert_id)
906
break
907
else: