@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ad44437ba3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[Bugfix] Fix Mamba model initialization and MLP Speculator weights loading (#10456)

Signed-off-by: Isotr0py <2037008807@qq.com>

Isotr0py committed Nov 20, 2024 at 13:04 UTC ad44437ba33e8d31962d272be238eeed4a1b4f84
2 files changed +4 -7
vllm/model_executor/models/mamba.py
+2 -6
@@ -1,5 +1,5 @@
1 """PyTorch MAMBA model."""
2 -from typing import Iterable, List, Optional, Set, Tuple
2 +from typing import Iterable, List, Optional, Tuple
3
4 import torch
5 from torch import nn
@@ -243,10 +243,8 @@ class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree):
243 next_tokens = self.sampler(logits, sampling_metadata)
244 return next_tokens
245
246 - def load_weights(self, weights: Iterable[Tuple[str,
247 - torch.Tensor]]) -> Set[str]:
246 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
247 params_dict = dict(self.named_parameters())
249 - loaded_params: Set[str] = set()
248 for name, loaded_weight in weights:
249 if "A_log" in name:
250 name = name.replace("A_log", "A")
@@ -258,5 +256,3 @@ class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree):
256 weight_loader = getattr(param, "weight_loader",
257 default_weight_loader)
258 weight_loader(param, loaded_weight)
261 - loaded_params.add(name)
262 - return loaded_params
vllm/model_executor/models/mlp_speculator.py
+2 -1
@@ -193,7 +193,8 @@ class MLPSpeculator(nn.Module):
193 params_dict = dict(self.named_parameters())
194 loaded_params: Set[str] = set()
195 for name, loaded_weight in weights:
196 - param = params_dict.get(name.replace("speculator.", ""))
196 + name = name.replace("speculator.", "")
197 + param = params_dict.get(name)
198 if param is not None:
199 weight_loader = getattr(param, "weight_loader",
200 default_weight_loader)