A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
[BugFix] Fix weight loading for Mixtral with TP (#2208)
Woosuk Kwon committed
Dec 19, 2023 at 16:16 UTC
ba4f82673884b644a516c2088087ef42688e202c
1 file changed
+5
-26
vllm/model_executor/models/mixtral.py
+5
-26
@@ -49,7 +49,6 @@ from vllm.model_executor.parallel_utils.parallel_state import (
49
from vllm.model_executor.sampling_metadata import SamplingMetadata
50
from vllm.model_executor.weight_utils import (default_weight_loader,
51
hf_model_weights_iterator)
52
-from vllm.model_executor.utils import set_weight_attrs
52
from vllm.sequence import SamplerOutput
53
54
KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -94,30 +93,6 @@ class MixtralMLP(nn.Module):
93
return current_hidden_states
94
95
97
-class DummyModule(nn.Module):
98
-
99
- def __init__(self) -> None:
100
- super().__init__()
101
-
102
- self.w1 = nn.Linear(0, 0, bias=False)
103
- self.w2 = nn.Linear(0, 0, bias=False)
104
- self.w3 = nn.Linear(0, 0, bias=False)
105
-
106
- set_weight_attrs(self.w1.weight,
107
- {"weight_loader": self.dummy_weight_loader})
108
- set_weight_attrs(self.w2.weight,
109
- {"weight_loader": self.dummy_weight_loader})
110
- set_weight_attrs(self.w3.weight,
111
- {"weight_loader": self.dummy_weight_loader})
112
-
113
- def forward(self, *args, **kwargs) -> None:
114
- raise NotImplementedError()
115
-
116
- def dummy_weight_loader(self, *args, **kwargs) -> None: # pylint: disable=unused-argument
117
- # Noop
118
- return
119
-
120
-
96
class MixtralMoE(nn.Module):
97
98
def __init__(
@@ -147,7 +122,7 @@ class MixtralMoE(nn.Module):
122
config.hidden_size,
123
config.intermediate_size,
124
linear_method=linear_method)
150
- if idx in self.expert_indicies else DummyModule()
125
+ if idx in self.expert_indicies else None
126
for idx in range(self.num_total_experts)
127
])
128
self.gate = ReplicatedLinear(config.hidden_size,
@@ -427,6 +402,10 @@ class MixtralForCausalLM(nn.Module):
402
# Skip loading extra bias for GPTQ models.
403
if name.endswith(".bias") and name not in params_dict:
404
continue
405
+ # Skip experts that are not assigned to this worker.
406
+ if ("block_sparse_moe.experts." in name
407
+ and name not in params_dict):
408
+ continue
409
param = params_dict[name]
410
weight_loader = getattr(param, "weight_loader",
411
default_weight_loader)