A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[Bugfix] Fix Mamba model initialization and MLP Speculator weights loading (#10456)
Signed-off-by: Isotr0py <2037008807@qq.com>
Isotr0py committed
Nov 20, 2024 at 13:04 UTC
ad44437ba33e8d31962d272be238eeed4a1b4f84
2 files changed
+4
-7
vllm/model_executor/models/mamba.py
+2
-6
@@ -1,5 +1,5 @@
1
"""PyTorch MAMBA model."""
2
-from typing import Iterable, List, Optional, Set, Tuple
2
+from typing import Iterable, List, Optional, Tuple
3
4
import torch
5
from torch import nn
@@ -243,10 +243,8 @@ class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree):
243
next_tokens = self.sampler(logits, sampling_metadata)
244
return next_tokens
245
246
- def load_weights(self, weights: Iterable[Tuple[str,
247
- torch.Tensor]]) -> Set[str]:
246
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
247
params_dict = dict(self.named_parameters())
249
- loaded_params: Set[str] = set()
248
for name, loaded_weight in weights:
249
if "A_log" in name:
250
name = name.replace("A_log", "A")
@@ -258,5 +256,3 @@ class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree):
256
weight_loader = getattr(param, "weight_loader",
257
default_weight_loader)
258
weight_loader(param, loaded_weight)
261
- loaded_params.add(name)
262
- return loaded_params
vllm/model_executor/models/mlp_speculator.py
+2
-1
@@ -193,7 +193,8 @@ class MLPSpeculator(nn.Module):
193
params_dict = dict(self.named_parameters())
194
loaded_params: Set[str] = set()
195
for name, loaded_weight in weights:
196
- param = params_dict.get(name.replace("speculator.", ""))
196
+ name = name.replace("speculator.", "")
197
+ param = params_dict.get(name)
198
if param is not None:
199
weight_loader = getattr(param, "weight_loader",
200
default_weight_loader)