@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 18ae428a0d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Bugfix] Fix Phi3.5 mini and MoE LoRA inference (#8571)

Amit Garg committed Sep 19, 2024 at 17:54 UTC 18ae428a0d8792d160d811a9cd5bb004d68ea8bd
3 files changed +22 -1
vllm/model_executor/models/__init__.py
+1 -1
@@ -50,7 +50,7 @@ _GENERATION_MODELS = {
50 "OrionForCausalLM": ("orion", "OrionForCausalLM"),
51 "PersimmonForCausalLM": ("persimmon", "PersimmonForCausalLM"),
52 "PhiForCausalLM": ("phi", "PhiForCausalLM"),
53 - "Phi3ForCausalLM": ("llama", "LlamaForCausalLM"),
53 + "Phi3ForCausalLM": ("phi3", "Phi3ForCausalLM"),
54 "PhiMoEForCausalLM": ("phimoe", "PhiMoEForCausalLM"),
55 "Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
56 "Qwen2MoeForCausalLM": ("qwen2_moe", "Qwen2MoeForCausalLM"),
vllm/model_executor/models/phi3.py new
+17
@@ -0,0 +1,17 @@
1 +# coding=utf-8
2 +# Adapted from llama.py
3 +"""Inference-only Phi3 model code inherit from Llama.py"""
4 +
5 +from vllm.model_executor.models.llama import LlamaForCausalLM
6 +
7 +
8 +class Phi3ForCausalLM(LlamaForCausalLM):
9 +
10 + packed_modules_mapping = {
11 + "qkv_proj": [
12 + "qkv_proj",
13 + ],
14 + "gate_up_proj": [
15 + "gate_up_proj",
16 + ],
17 + }
vllm/model_executor/models/phimoe.py
+4
@@ -491,6 +491,10 @@ class PhiMoEForCausalLM(nn.Module, SupportsLoRA):
491 "o_proj",
492 "embed_tokens",
493 "lm_head",
494 + "w1",
495 + "w2",
496 + "w3",
497 + "gate",
498 ]
499 embedding_modules = {
500 "embed_tokens": "input_embeddings",