@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f12c3b5b3d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Model] Add Phi-2 LoRA support (#4886)

Isotr0py committed May 21, 2024 at 13:24 UTC f12c3b5b3d076a67662b76d215fd875fd6cdf6d7
4 files changed +100 -7
docs/source/models/supported_models.rst
+1 -1
@@ -118,7 +118,7 @@ Alongside each architecture, we include some popular models that use it.
118 * - :code:`PhiForCausalLM`
119 - Phi
120 - :code:`microsoft/phi-1_5`, :code:`microsoft/phi-2`, etc.
121 - -
121 + - ✅︎
122 * - :code:`Phi3ForCausalLM`
123 - Phi-3
124 - :code:`microsoft/Phi-3-mini-4k-instruct`, :code:`microsoft/Phi-3-mini-128k-instruct`, etc.
tests/lora/conftest.py
+5
@@ -165,6 +165,11 @@ def tinyllama_lora_files():
165 return snapshot_download(repo_id="jashing/tinyllama-colorist-lora")
166
167
168 +@pytest.fixture(scope="session")
169 +def phi2_lora_files():
170 + return snapshot_download(repo_id="isotr0py/phi-2-test-sql-lora")
171 +
172 +
173 @pytest.fixture(scope="session")
174 def long_context_lora_files_16k_1():
175 return snapshot_download(repo_id="SangBinCho/long_context_16k_testing_1")
tests/lora/test_phi.py new
+67
@@ -0,0 +1,67 @@
1 +import vllm
2 +from vllm.lora.request import LoRARequest
3 +
4 +MODEL_PATH = "microsoft/phi-2"
5 +
6 +PROMPT_TEMPLATE = "### Instruct: {sql_prompt}\n\n### Context: {context}\n\n### Output:" # noqa: E501
7 +
8 +
9 +def do_sample(llm, lora_path: str, lora_id: int) -> str:
10 + prompts = [
11 + PROMPT_TEMPLATE.format(
12 + sql_prompt=
13 + "Which catalog publisher has published the most catalogs?",
14 + context="CREATE TABLE catalogs (catalog_publisher VARCHAR);"),
15 + PROMPT_TEMPLATE.format(
16 + sql_prompt=
17 + "Which trip started from the station with the largest dock count? Give me the trip id.", # noqa: E501
18 + context=
19 + "CREATE TABLE trip (id VARCHAR, start_station_id VARCHAR); CREATE TABLE station (id VARCHAR, dock_count VARCHAR);" # noqa: E501
20 + ),
21 + PROMPT_TEMPLATE.format(
22 + sql_prompt=
23 + "How many marine species are found in the Southern Ocean?", # noqa: E501
24 + context=
25 + "CREATE TABLE marine_species (name VARCHAR(50), common_name VARCHAR(50), location VARCHAR(50));" # noqa: E501
26 + ),
27 + ]
28 + sampling_params = vllm.SamplingParams(temperature=0,
29 + max_tokens=64,
30 + stop="### End")
31 + outputs = llm.generate(
32 + prompts,
33 + sampling_params,
34 + lora_request=LoRARequest(str(lora_id), lora_id, lora_path)
35 + if lora_id else None,
36 + )
37 + # Print the outputs.
38 + generated_texts = []
39 + for output in outputs:
40 + prompt = output.prompt
41 + generated_text = output.outputs[0].text.strip()
42 + generated_texts.append(generated_text)
43 + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
44 + return generated_texts
45 +
46 +
47 +def test_phi2_lora(phi2_lora_files):
48 + # We enable enforce_eager=True here to reduce VRAM usage for lora-test CI,
49 + # Otherwise, the lora-test will fail due to CUDA OOM.
50 + llm = vllm.LLM(MODEL_PATH,
51 + max_model_len=1024,
52 + enable_lora=True,
53 + max_loras=2,
54 + enforce_eager=True)
55 +
56 + expected_lora_output = [
57 + "SELECT catalog_publisher, COUNT(*) as num_catalogs FROM catalogs GROUP BY catalog_publisher ORDER BY num_catalogs DESC LIMIT 1;", # noqa: E501
58 + "SELECT trip.id FROM trip JOIN station ON trip.start_station_id = station.id WHERE station.dock_count = (SELECT MAX(dock_count) FROM station);", # noqa: E501
59 + "SELECT COUNT(*) FROM marine_species WHERE location = 'Southern Ocean';", # noqa: E501
60 + ]
61 +
62 + output1 = do_sample(llm, phi2_lora_files, lora_id=1)
63 + for i in range(len(expected_lora_output)):
64 + assert output1[i].startswith(expected_lora_output[i])
65 + output2 = do_sample(llm, phi2_lora_files, lora_id=2)
66 + for i in range(len(expected_lora_output)):
67 + assert output2[i].startswith(expected_lora_output[i])
vllm/model_executor/models/phi.py
+27 -6
@@ -42,7 +42,7 @@ from torch import nn
42 from transformers import PretrainedConfig
43
44 from vllm.attention import Attention, AttentionMetadata
45 -from vllm.config import CacheConfig
45 +from vllm.config import CacheConfig, LoRAConfig
46 from vllm.distributed import get_tensor_model_parallel_world_size
47 from vllm.model_executor.layers.activation import get_act_fn
48 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -229,11 +229,32 @@ class PhiModel(nn.Module):
229
230
231 class PhiForCausalLM(nn.Module):
232 -
233 - def __init__(self,
234 - config: PretrainedConfig,
235 - cache_config: Optional[CacheConfig] = None,
236 - quant_config: Optional[QuantizationConfig] = None):
232 + packed_modules_mapping = {
233 + "qkv_proj": [
234 + "q_proj",
235 + "k_proj",
236 + "v_proj",
237 + ]
238 + }
239 +
240 + # LoRA specific attributes
241 + supported_lora_modules = [
242 + "qkv_proj",
243 + "dense",
244 + "fc1",
245 + "fc2",
246 + ]
247 + embedding_modules = {}
248 + embedding_padding_modules = []
249 +
250 + def __init__(
251 + self,
252 + config: PretrainedConfig,
253 + cache_config: Optional[CacheConfig] = None,
254 + quant_config: Optional[QuantizationConfig] = None,
255 + lora_config: Optional[LoRAConfig] = None,
256 + ):
257 + del lora_config # Unused.
258 super().__init__()
259 self.config = config
260 self.quant_config = quant_config