A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Model] add a lora module for granite 3.0 MoE models (#9673)
Will Johnson committed
Oct 25, 2024 at 01:00 UTC
a6f37218619df39760624d541bf7911ab911f792
1 file changed
+1
vllm/model_executor/models/granitemoe.py
+1
@@ -324,6 +324,7 @@ class GraniteMoeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
324
"o_proj",
325
"embed_tokens",
326
"lm_head",
327
+ "layer",
328
]
329
embedding_modules = {
330
"embed_tokens": "input_embeddings",