A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Bugfix] Fix Phi3.5 mini and MoE LoRA inference (#8571)
Amit Garg committed
Sep 19, 2024 at 17:54 UTC
18ae428a0d8792d160d811a9cd5bb004d68ea8bd
3 files changed
+22
-1
vllm/model_executor/models/__init__.py
+1
-1
@@ -50,7 +50,7 @@ _GENERATION_MODELS = {
50
"OrionForCausalLM": ("orion", "OrionForCausalLM"),
51
"PersimmonForCausalLM": ("persimmon", "PersimmonForCausalLM"),
52
"PhiForCausalLM": ("phi", "PhiForCausalLM"),
53
- "Phi3ForCausalLM": ("llama", "LlamaForCausalLM"),
53
+ "Phi3ForCausalLM": ("phi3", "Phi3ForCausalLM"),
54
"PhiMoEForCausalLM": ("phimoe", "PhiMoEForCausalLM"),
55
"Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
56
"Qwen2MoeForCausalLM": ("qwen2_moe", "Qwen2MoeForCausalLM"),
vllm/model_executor/models/phi3.py
new
+17
@@ -0,0 +1,17 @@
1
+# coding=utf-8
2
+# Adapted from llama.py
3
+"""Inference-only Phi3 model code inherit from Llama.py"""
4
+
5
+from vllm.model_executor.models.llama import LlamaForCausalLM
6
+
7
+
8
+class Phi3ForCausalLM(LlamaForCausalLM):
9
+
10
+ packed_modules_mapping = {
11
+ "qkv_proj": [
12
+ "qkv_proj",
13
+ ],
14
+ "gate_up_proj": [
15
+ "gate_up_proj",
16
+ ],
17
+ }
vllm/model_executor/models/phimoe.py
+4
@@ -491,6 +491,10 @@ class PhiMoEForCausalLM(nn.Module, SupportsLoRA):
491
"o_proj",
492
"embed_tokens",
493
"lm_head",
494
+ "w1",
495
+ "w2",
496
+ "w3",
497
+ "gate",
498
]
499
embedding_modules = {
500
"embed_tokens": "input_embeddings",