A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 12:15 UTC
[Model] Ignore rotary embed load for Cohere model (#17319)
Ekagra Ranjan committed
Apr 29, 2025 at 03:30 UTC
97cc8729f0bc351a5536380fd897607f4ecdeef1
1 file changed
+4
vllm/model_executor/models/commandr.py
+4
@@ -418,6 +418,10 @@ class CohereForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsQuant):
418
loaded_params: Set[str] = set()
419
for name, loaded_weight in weights:
420
421
+ # Skip loading rotary embeddings since vLLM has its own
422
+ if "rotary_emb.inv_freq" in name:
423
+ continue
424
+
425
if (self.quant_config is not None and
426
(scale_name := self.quant_config.get_cache_scale(name))):
427
# Loading kv cache quantization scales