A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Fix tie_word_embeddings for Qwen2. (#3344)
Yang Fan committed
Mar 16, 2024 at 00:36 UTC
a7c871680e622d998c9a4585404b0b88bad9ba92
1 file changed
+12
-2
vllm/model_executor/models/qwen2.py
+12
-2
@@ -299,7 +299,11 @@ class Qwen2ForCausalLM(nn.Module):
299
self.config = config
300
self.linear_method = linear_method
301
self.model = Qwen2Model(config, linear_method)
302
- self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
302
+
303
+ if not config.tie_word_embeddings:
304
+ self.lm_head = ParallelLMHead(config.vocab_size,
305
+ config.hidden_size)
306
+
307
self.sampler = Sampler(config.vocab_size)
308
309
def forward(
@@ -318,7 +322,11 @@ class Qwen2ForCausalLM(nn.Module):
322
hidden_states: torch.Tensor,
323
sampling_metadata: SamplingMetadata,
324
) -> Optional[SamplerOutput]:
321
- next_tokens = self.sampler(self.lm_head.weight, hidden_states,
325
+ if self.config.tie_word_embeddings:
326
+ lm_head_weight = self.model.embed_tokens.weight
327
+ else:
328
+ lm_head_weight = self.lm_head.weight
329
+ next_tokens = self.sampler(lm_head_weight, hidden_states,
330
sampling_metadata)
331
return next_tokens
332
@@ -340,6 +348,8 @@ class Qwen2ForCausalLM(nn.Module):
348
model_name_or_path, cache_dir, load_format, revision):
349
if "rotary_emb.inv_freq" in name:
350
continue
351
+ if self.config.tie_word_embeddings and "lm_head.weight" in name:
352
+ continue
353
for (param_name, weight_name, shard_id) in stacked_params_mapping:
354
if weight_name not in name:
355
continue