A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Model] Add Phi-2 LoRA support (#4886)
Isotr0py committed
May 21, 2024 at 13:24 UTC
f12c3b5b3d076a67662b76d215fd875fd6cdf6d7
4 files changed
+100
-7
docs/source/models/supported_models.rst
+1
-1
@@ -118,7 +118,7 @@ Alongside each architecture, we include some popular models that use it.
118
* - :code:`PhiForCausalLM`
119
- Phi
120
- :code:`microsoft/phi-1_5`, :code:`microsoft/phi-2`, etc.
121
- -
121
+ - ✅︎
122
* - :code:`Phi3ForCausalLM`
123
- Phi-3
124
- :code:`microsoft/Phi-3-mini-4k-instruct`, :code:`microsoft/Phi-3-mini-128k-instruct`, etc.
tests/lora/conftest.py
+5
@@ -165,6 +165,11 @@ def tinyllama_lora_files():
165
return snapshot_download(repo_id="jashing/tinyllama-colorist-lora")
166
167
168
+@pytest.fixture(scope="session")
169
+def phi2_lora_files():
170
+ return snapshot_download(repo_id="isotr0py/phi-2-test-sql-lora")
171
+
172
+
173
@pytest.fixture(scope="session")
174
def long_context_lora_files_16k_1():
175
return snapshot_download(repo_id="SangBinCho/long_context_16k_testing_1")
tests/lora/test_phi.py
new
+67
@@ -0,0 +1,67 @@
1
+import vllm
2
+from vllm.lora.request import LoRARequest
3
+
4
+MODEL_PATH = "microsoft/phi-2"
5
+
6
+PROMPT_TEMPLATE = "### Instruct: {sql_prompt}\n\n### Context: {context}\n\n### Output:" # noqa: E501
7
+
8
+
9
+def do_sample(llm, lora_path: str, lora_id: int) -> str:
10
+ prompts = [
11
+ PROMPT_TEMPLATE.format(
12
+ sql_prompt=
13
+ "Which catalog publisher has published the most catalogs?",
14
+ context="CREATE TABLE catalogs (catalog_publisher VARCHAR);"),
15
+ PROMPT_TEMPLATE.format(
16
+ sql_prompt=
17
+ "Which trip started from the station with the largest dock count? Give me the trip id.", # noqa: E501
18
+ context=
19
+ "CREATE TABLE trip (id VARCHAR, start_station_id VARCHAR); CREATE TABLE station (id VARCHAR, dock_count VARCHAR);" # noqa: E501
20
+ ),
21
+ PROMPT_TEMPLATE.format(
22
+ sql_prompt=
23
+ "How many marine species are found in the Southern Ocean?", # noqa: E501
24
+ context=
25
+ "CREATE TABLE marine_species (name VARCHAR(50), common_name VARCHAR(50), location VARCHAR(50));" # noqa: E501
26
+ ),
27
+ ]
28
+ sampling_params = vllm.SamplingParams(temperature=0,
29
+ max_tokens=64,
30
+ stop="### End")
31
+ outputs = llm.generate(
32
+ prompts,
33
+ sampling_params,
34
+ lora_request=LoRARequest(str(lora_id), lora_id, lora_path)
35
+ if lora_id else None,
36
+ )
37
+ # Print the outputs.
38
+ generated_texts = []
39
+ for output in outputs:
40
+ prompt = output.prompt
41
+ generated_text = output.outputs[0].text.strip()
42
+ generated_texts.append(generated_text)
43
+ print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
44
+ return generated_texts
45
+
46
+
47
+def test_phi2_lora(phi2_lora_files):
48
+ # We enable enforce_eager=True here to reduce VRAM usage for lora-test CI,
49
+ # Otherwise, the lora-test will fail due to CUDA OOM.
50
+ llm = vllm.LLM(MODEL_PATH,
51
+ max_model_len=1024,
52
+ enable_lora=True,
53
+ max_loras=2,
54
+ enforce_eager=True)
55
+
56
+ expected_lora_output = [
57
+ "SELECT catalog_publisher, COUNT(*) as num_catalogs FROM catalogs GROUP BY catalog_publisher ORDER BY num_catalogs DESC LIMIT 1;", # noqa: E501
58
+ "SELECT trip.id FROM trip JOIN station ON trip.start_station_id = station.id WHERE station.dock_count = (SELECT MAX(dock_count) FROM station);", # noqa: E501
59
+ "SELECT COUNT(*) FROM marine_species WHERE location = 'Southern Ocean';", # noqa: E501
60
+ ]
61
+
62
+ output1 = do_sample(llm, phi2_lora_files, lora_id=1)
63
+ for i in range(len(expected_lora_output)):
64
+ assert output1[i].startswith(expected_lora_output[i])
65
+ output2 = do_sample(llm, phi2_lora_files, lora_id=2)
66
+ for i in range(len(expected_lora_output)):
67
+ assert output2[i].startswith(expected_lora_output[i])
vllm/model_executor/models/phi.py
+27
-6
@@ -42,7 +42,7 @@ from torch import nn
42
from transformers import PretrainedConfig
43
44
from vllm.attention import Attention, AttentionMetadata
45
-from vllm.config import CacheConfig
45
+from vllm.config import CacheConfig, LoRAConfig
46
from vllm.distributed import get_tensor_model_parallel_world_size
47
from vllm.model_executor.layers.activation import get_act_fn
48
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -229,11 +229,32 @@ class PhiModel(nn.Module):
229
230
231
class PhiForCausalLM(nn.Module):
232
-
233
- def __init__(self,
234
- config: PretrainedConfig,
235
- cache_config: Optional[CacheConfig] = None,
236
- quant_config: Optional[QuantizationConfig] = None):
232
+ packed_modules_mapping = {
233
+ "qkv_proj": [
234
+ "q_proj",
235
+ "k_proj",
236
+ "v_proj",
237
+ ]
238
+ }
239
+
240
+ # LoRA specific attributes
241
+ supported_lora_modules = [
242
+ "qkv_proj",
243
+ "dense",
244
+ "fc1",
245
+ "fc2",
246
+ ]
247
+ embedding_modules = {}
248
+ embedding_padding_modules = []
249
+
250
+ def __init__(
251
+ self,
252
+ config: PretrainedConfig,
253
+ cache_config: Optional[CacheConfig] = None,
254
+ quant_config: Optional[QuantizationConfig] = None,
255
+ lora_config: Optional[LoRAConfig] = None,
256
+ ):
257
+ del lora_config # Unused.
258
super().__init__()
259
self.config = config
260
self.quant_config = quant_config