A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[Bugfix] Missing quant_config in deepseek embedding layer (#12836)
Szymon Ożóg committed
Feb 7, 2025 at 06:35 UTC
aa375dca9fbeff03904cd7b7dcc5014bfa19b0fb
1 file changed
+2
-1
vllm/model_executor/models/deepseek_v2.py
+2
-1
@@ -581,7 +581,8 @@ class DeepseekV2Model(nn.Module):
581
self.embed_tokens = VocabParallelEmbedding(
582
config.vocab_size,
583
config.hidden_size,
584
- )
584
+ quant_config=quant_config,
585
+ prefix=f"{prefix}.embed_tokens")
586
else:
587
self.embed_tokens = PPMissingLayer()
588