@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ba4f826738
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

[BugFix] Fix weight loading for Mixtral with TP (#2208)

Woosuk Kwon committed Dec 19, 2023 at 16:16 UTC ba4f82673884b644a516c2088087ef42688e202c
1 file changed +5 -26
vllm/model_executor/models/mixtral.py
+5 -26
@@ -49,7 +49,6 @@ from vllm.model_executor.parallel_utils.parallel_state import (
49 from vllm.model_executor.sampling_metadata import SamplingMetadata
50 from vllm.model_executor.weight_utils import (default_weight_loader,
51 hf_model_weights_iterator)
52 -from vllm.model_executor.utils import set_weight_attrs
52 from vllm.sequence import SamplerOutput
53
54 KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -94,30 +93,6 @@ class MixtralMLP(nn.Module):
93 return current_hidden_states
94
95
97 -class DummyModule(nn.Module):
98 -
99 - def __init__(self) -> None:
100 - super().__init__()
101 -
102 - self.w1 = nn.Linear(0, 0, bias=False)
103 - self.w2 = nn.Linear(0, 0, bias=False)
104 - self.w3 = nn.Linear(0, 0, bias=False)
105 -
106 - set_weight_attrs(self.w1.weight,
107 - {"weight_loader": self.dummy_weight_loader})
108 - set_weight_attrs(self.w2.weight,
109 - {"weight_loader": self.dummy_weight_loader})
110 - set_weight_attrs(self.w3.weight,
111 - {"weight_loader": self.dummy_weight_loader})
112 -
113 - def forward(self, *args, **kwargs) -> None:
114 - raise NotImplementedError()
115 -
116 - def dummy_weight_loader(self, *args, **kwargs) -> None: # pylint: disable=unused-argument
117 - # Noop
118 - return
119 -
120 -
96 class MixtralMoE(nn.Module):
97
98 def __init__(
@@ -147,7 +122,7 @@ class MixtralMoE(nn.Module):
122 config.hidden_size,
123 config.intermediate_size,
124 linear_method=linear_method)
150 - if idx in self.expert_indicies else DummyModule()
125 + if idx in self.expert_indicies else None
126 for idx in range(self.num_total_experts)
127 ])
128 self.gate = ReplicatedLinear(config.hidden_size,
@@ -427,6 +402,10 @@ class MixtralForCausalLM(nn.Module):
402 # Skip loading extra bias for GPTQ models.
403 if name.endswith(".bias") and name not in params_dict:
404 continue
405 + # Skip experts that are not assigned to this worker.
406 + if ("block_sparse_moe.experts." in name
407 + and name not in params_dict):
408 + continue
409 param = params_dict[name]
410 weight_loader = getattr(param, "weight_loader",
411 default_weight_loader)