@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 24cde76a15
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

[Minor] Add comment on skipping rope caches (#2004)

Woosuk Kwon committed Dec 10, 2023 at 10:04 UTC 24cde76a152fbffde30fa2be0d08dcbad490530e
1 file changed +4 -3
vllm/model_executor/models/llama.py
+4 -3
@@ -322,9 +322,10 @@ class LlamaForCausalLM(nn.Module):
322 model_name_or_path, cache_dir, load_format, revision):
323 if "rotary_emb.inv_freq" in name:
324 continue
325 - if "rotary_emb.cos_cached" in name:
326 - continue
327 - if "rotary_emb.sin_cached" in name:
325 + if ("rotary_emb.cos_cached" in name
326 + or "rotary_emb.sin_cached" in name):
327 + # Models trained using ColossalAI may include these tensors in
328 + # the checkpoint. Skip them.
329 continue
330 for (param_name, weight_name, shard_id) in stacked_params_mapping:
331 if weight_name not in name: