@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 343041c4c4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[model] Reduce medusa weight (#10454)

Signed-off-by: skylee-01 <497627264@qq.com>

Sky Lee committed Nov 20, 2024 at 14:05 UTC 343041c4c4db93b4693ba437df7ae8bea485d18e
1 file changed +18 -4
vllm/model_executor/models/medusa.py
+18 -4
@@ -61,14 +61,25 @@ class Medusa(nn.Module):
61 self.truncated_vocab_size = config.truncated_vocab_size
62 self.unpadded_vocab_size = self.truncated_vocab_size
63
64 - self.lm_heads = nn.ModuleList([
65 - ParallelLMHead(
64 + if getattr(config, "original_lm_head", False):
65 + self.lm_head = ParallelLMHead(
66 self.unpadded_vocab_size,
67 config.hidden_size,
68 org_num_embeddings=self.truncated_vocab_size,
69 padding_size=DEFAULT_VOCAB_PADDING_SIZE,
70 - ) for _ in range(self.config.num_heads)
71 - ])
70 + )
71 + self.lm_heads = [
72 + self.lm_head for _ in range(self.config.num_heads)
73 + ]
74 + else:
75 + self.lm_heads = nn.ModuleList([
76 + ParallelLMHead(
77 + self.unpadded_vocab_size,
78 + config.hidden_size,
79 + org_num_embeddings=self.truncated_vocab_size,
80 + padding_size=DEFAULT_VOCAB_PADDING_SIZE,
81 + ) for _ in range(self.config.num_heads)
82 + ])
83
84 logit_scale = getattr(config, "logit_scale", 1.0)
85 self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
@@ -172,6 +183,9 @@ class Medusa(nn.Module):
183 requires_grad=False)
184 elif name in params_dict:
185 weights_map[name] = loaded_weight
186 + elif (getattr(self.config, "original_lm_head", False)
187 + and name == "lm_heads.0.weight"):
188 + weights_map["lm_head.weight"] = loaded_weight
189
190 for name, loaded_weight in weights_map.items():
191 if "lm_head" in name and self.token_map is not None and\