@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a7c871680e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Fix tie_word_embeddings for Qwen2. (#3344)

Yang Fan committed Mar 16, 2024 at 00:36 UTC a7c871680e622d998c9a4585404b0b88bad9ba92
1 file changed +12 -2
vllm/model_executor/models/qwen2.py
+12 -2
@@ -299,7 +299,11 @@ class Qwen2ForCausalLM(nn.Module):
299 self.config = config
300 self.linear_method = linear_method
301 self.model = Qwen2Model(config, linear_method)
302 - self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
302 +
303 + if not config.tie_word_embeddings:
304 + self.lm_head = ParallelLMHead(config.vocab_size,
305 + config.hidden_size)
306 +
307 self.sampler = Sampler(config.vocab_size)
308
309 def forward(
@@ -318,7 +322,11 @@ class Qwen2ForCausalLM(nn.Module):
322 hidden_states: torch.Tensor,
323 sampling_metadata: SamplingMetadata,
324 ) -> Optional[SamplerOutput]:
321 - next_tokens = self.sampler(self.lm_head.weight, hidden_states,
325 + if self.config.tie_word_embeddings:
326 + lm_head_weight = self.model.embed_tokens.weight
327 + else:
328 + lm_head_weight = self.lm_head.weight
329 + next_tokens = self.sampler(lm_head_weight, hidden_states,
330 sampling_metadata)
331 return next_tokens
332
@@ -340,6 +348,8 @@ class Qwen2ForCausalLM(nn.Module):
348 model_name_or_path, cache_dir, load_format, revision):
349 if "rotary_emb.inv_freq" in name:
350 continue
351 + if self.config.tie_word_embeddings and "lm_head.weight" in name:
352 + continue
353 for (param_name, weight_name, shard_id) in stacked_params_mapping:
354 if weight_name not in name:
355 continue