A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[Bugfix] Fix LoRA weight sharding (#10450)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
Jee Jee Li committed
Nov 24, 2024 at 09:23 UTC
1700c543a556e669e559c369a36c0a0d36a8de19
7 files changed
+258
-168
.buildkite/test-pipeline.yaml
+9
-4
@@ -230,7 +230,7 @@ steps:
230
source_file_dependencies:
231
- vllm/lora
232
- tests/lora
233
- command: pytest -v -s lora --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --ignore=lora/test_long_context.py
233
+ command: pytest -v -s lora --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --ignore lora/test_long_context.py lora/test_chatglm3_tp.py lora/test_llama_tp.py
234
parallelism: 4
235
236
- label: "PyTorch Fullgraph Smoke Test" # 9min
@@ -475,18 +475,23 @@ steps:
475
- pytest -v -s distributed/test_pp_cudagraph.py
476
- pytest -v -s distributed/test_pipeline_parallel.py
477
478
-- label: LoRA Long Context (Distributed) # 11min
479
- # This test runs llama 13B, so it is required to run on 4 GPUs.
478
+- label: LoRA TP Test (Distributed)
479
num_gpus: 4
480
soft_fail: true
481
source_file_dependencies:
482
- vllm/lora
484
- - tests/lora/test_long_context
483
+ - tests/lora
484
commands:
485
# FIXIT: find out which code initialize cuda before running the test
486
# before the fix, we need to use spawn to test it
487
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
488
+ # This test runs llama 13B, so it is required to run on 4 GPUs.
489
- pytest -v -s -x lora/test_long_context.py
490
+ # There is some Tensor Parallelism related processing logic in LoRA that
491
+ # requires multi-GPU testing for validation.
492
+ - pytest -v -s -x lora/test_chatglm3_tp.py
493
+ - pytest -v -s -x lora/test_llama_tp.py
494
+
495
496
- label: Weight Loading Multiple GPU Test # 33min
497
working_dir: "/vllm-workspace/tests"
tests/lora/test_chatglm3_tp.py
renamed
+53
-10
@@ -1,12 +1,21 @@
1
from typing import List
2
3
import vllm
4
+from tests.utils import fork_new_process_for_each_test
5
from vllm.lora.request import LoRARequest
6
7
+from ..utils import multi_gpu_test
8
+
9
MODEL_PATH = "THUDM/chatglm3-6b"
10
11
PROMPT_TEMPLATE = """I want you to act as a SQL terminal in front of an example database, you need only to return the sql command to me.Below is an instruction that describes a task, Write a response that appropriately completes the request.\n"\n##Instruction:\nconcert_singer contains tables such as stadium, singer, concert, singer_in_concert. Table stadium has columns such as Stadium_ID, Location, Name, Capacity, Highest, Lowest, Average. Stadium_ID is the primary key.\nTable singer has columns such as Singer_ID, Name, Country, Song_Name, Song_release_year, Age, Is_male. Singer_ID is the primary key.\nTable concert has columns such as concert_ID, concert_Name, Theme, Stadium_ID, Year. concert_ID is the primary key.\nTable singer_in_concert has columns such as concert_ID, Singer_ID. concert_ID is the primary key.\nThe Stadium_ID of concert is the foreign key of Stadium_ID of stadium.\nThe Singer_ID of singer_in_concert is the foreign key of Singer_ID of singer.\nThe concert_ID of singer_in_concert is the foreign key of concert_ID of concert.\n\n###Input:\n{query}\n\n###Response:""" # noqa: E501
12
13
+EXPECTED_LORA_OUTPUT = [
14
+ "SELECT count(*) FROM singer",
15
+ "SELECT avg(age) , min(age) , max(age) FROM singer WHERE country = 'France'", # noqa: E501
16
+ "SELECT name , country , age FROM singer ORDER BY age",
17
+]
18
+
19
20
def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:
21
prompts = [
@@ -20,7 +29,6 @@ def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:
29
"Show name, country, age for all singers ordered by age from the oldest to the youngest." # noqa: E501
30
),
31
]
23
- print(prompts)
32
sampling_params = vllm.SamplingParams(temperature=0, max_tokens=32)
33
outputs = llm.generate(
34
prompts,
@@ -37,23 +45,58 @@ def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:
45
return generated_texts
46
47
48
+@fork_new_process_for_each_test
49
def test_chatglm3_lora(chatglm3_lora_files):
50
llm = vllm.LLM(MODEL_PATH,
51
max_model_len=1024,
52
enable_lora=True,
53
max_loras=4,
54
max_lora_rank=64,
55
+ tensor_parallel_size=1,
56
trust_remote_code=True)
57
48
- expected_lora_output = [
49
- "SELECT count(*) FROM singer",
50
- "SELECT avg(age) , min(age) , max(age) FROM singer WHERE country = 'France'", # noqa: E501
51
- "SELECT name , country , age FROM singer ORDER BY age",
52
- ]
58
+ output1 = do_sample(llm, chatglm3_lora_files, lora_id=1)
59
+ for i in range(len(EXPECTED_LORA_OUTPUT)):
60
+ assert output1[i] == EXPECTED_LORA_OUTPUT[i]
61
+ output2 = do_sample(llm, chatglm3_lora_files, lora_id=2)
62
+ for i in range(len(EXPECTED_LORA_OUTPUT)):
63
+ assert output2[i] == EXPECTED_LORA_OUTPUT[i]
64
+
65
66
+@multi_gpu_test(num_gpus=4)
67
+@fork_new_process_for_each_test
68
+def test_chatglm3_lora_tp4(chatglm3_lora_files):
69
+ llm = vllm.LLM(MODEL_PATH,
70
+ max_model_len=1024,
71
+ enable_lora=True,
72
+ max_loras=4,
73
+ max_lora_rank=64,
74
+ tensor_parallel_size=4,
75
+ trust_remote_code=True,
76
+ fully_sharded_loras=False)
77
+
78
+ output1 = do_sample(llm, chatglm3_lora_files, lora_id=1)
79
+ for i in range(len(EXPECTED_LORA_OUTPUT)):
80
+ assert output1[i] == EXPECTED_LORA_OUTPUT[i]
81
+ output2 = do_sample(llm, chatglm3_lora_files, lora_id=2)
82
+ for i in range(len(EXPECTED_LORA_OUTPUT)):
83
+ assert output2[i] == EXPECTED_LORA_OUTPUT[i]
84
+
85
+
86
+@multi_gpu_test(num_gpus=4)
87
+@fork_new_process_for_each_test
88
+def test_chatglm3_lora_tp4_fully_sharded_loras(chatglm3_lora_files):
89
+ llm = vllm.LLM(MODEL_PATH,
90
+ max_model_len=1024,
91
+ enable_lora=True,
92
+ max_loras=4,
93
+ max_lora_rank=64,
94
+ tensor_parallel_size=4,
95
+ trust_remote_code=True,
96
+ fully_sharded_loras=True)
97
output1 = do_sample(llm, chatglm3_lora_files, lora_id=1)
55
- for i in range(len(expected_lora_output)):
56
- assert output1[i] == expected_lora_output[i]
98
+ for i in range(len(EXPECTED_LORA_OUTPUT)):
99
+ assert output1[i] == EXPECTED_LORA_OUTPUT[i]
100
output2 = do_sample(llm, chatglm3_lora_files, lora_id=2)
58
- for i in range(len(expected_lora_output)):
59
- assert output2[i] == expected_lora_output[i]
101
+ for i in range(len(EXPECTED_LORA_OUTPUT)):
102
+ assert output2[i] == EXPECTED_LORA_OUTPUT[i]
tests/lora/test_llama.py
deleted
-146
@@ -1,146 +0,0 @@
1
-from typing import List
2
-
3
-import pytest
4
-import ray
5
-
6
-import vllm
7
-from vllm.distributed import cleanup_dist_env_and_memory
8
-from vllm.lora.request import LoRARequest
9
-
10
-MODEL_PATH = "meta-llama/Llama-2-7b-hf"
11
-
12
-
13
-def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:
14
- prompts = [
15
- "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_74 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]", # noqa: E501
16
- "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_11 (nationality VARCHAR, elector VARCHAR)\n\n question: When Anchero Pantaleone was the elector what is under nationality? [/user] [assistant]", # noqa: E501
17
- "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_95 (one_mora VARCHAR, gloss VARCHAR, accented_mora VARCHAR)\n\n question: What is the one mora for a low tone mora with a gloss of /˩okiru/ [òkìɽɯ́]? [/user] [assistant]", # noqa: E501
18
- "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE candidate (people_id VARCHAR, unsure_rate INTEGER); CREATE TABLE people (sex VARCHAR, people_id VARCHAR)\n\n question: which gender got the highest average uncertain ratio. [/user] [assistant]", # noqa: E501
19
- "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_60 (pick INTEGER, former_wnba_team VARCHAR)\n\n question: What pick was a player that previously played for the Minnesota Lynx? [/user] [assistant]", # noqa: E501
20
- "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_28138035_4 (womens_doubles VARCHAR, mens_singles VARCHAR)\n\n question: Name the women's doubles for werner schlager [/user] [assistant]" # noqa: E501
21
- ]
22
- sampling_params = vllm.SamplingParams(temperature=0,
23
- max_tokens=256,
24
- stop=["[/assistant]"])
25
- outputs = llm.generate(
26
- prompts,
27
- sampling_params,
28
- lora_request=LoRARequest(str(lora_id), lora_id, lora_path)
29
- if lora_id else None)
30
- # Print the outputs.
31
- generated_texts: List[str] = []
32
- for output in outputs:
33
- prompt = output.prompt
34
- generated_text = output.outputs[0].text
35
- generated_texts.append(generated_text)
36
- print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
37
- return generated_texts
38
-
39
-
40
-@pytest.mark.parametrize("tp_size", [1, 2, 4])
41
-def test_llama_lora(sql_lora_files, tp_size, num_gpus_available):
42
- if num_gpus_available < tp_size:
43
- pytest.skip(f"Not enough GPUs for tensor parallelism {tp_size}")
44
-
45
- llm = vllm.LLM(MODEL_PATH,
46
- enable_lora=True,
47
- max_num_seqs=16,
48
- max_loras=4,
49
- tensor_parallel_size=tp_size)
50
-
51
- expected_no_lora_output = [
52
- "\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_75 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_76 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_77 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_78 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user]", # noqa: E501
53
- " Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_11 (nationality VARCHAR, elector VARCHAR)\n\n question: When Anchero Pantaleone was the elector what is under nationality? ", # noqa: E501
54
- "\n\n answer: 1\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_96 (one_mora VARCHAR, gloss VARCHAR, accented_mora VARCHAR)\n\n question: What is the one mora for a high tone mora with a gloss of /˧kot/ [kòt]? [/user] [assistant]\n\n answer: 2\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_97 (one_mora VARCHAR, gloss VARCHAR, accented_mora VARCHAR)\n\n question: What is the one mora for a high tone mora with a gloss of /˧kot/ [kòt]? [/user] [assistant]\n\n answer: 2\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_98 (one_mora VARCHAR, gloss VARCHAR, accented_mora VARCHAR)\n\n question: What is the one m", # noqa: E501
55
- " Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE candidate (people_id VARCHAR, unsure_rate INTEGER); CREATE TABLE people (sex VARCHAR, people_id VARCHAR)\n\n question: which gender got the highest average uncertain ratio. ", # noqa: E501
56
- " Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_60 (pick INTEGER, former_wnba_team VARCHAR)\n\n question: What pick was a player that previously played for the Minnesota Lynx? ", # noqa: E501
57
- "\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_28138035_4 (womens_doubles VARCHAR, mens_singles VARCHAR)\n\n question: Name the women's doubles for werner schlager [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_28138035_4 (womens_doubles VARCHAR, mens_singles VARCHAR)\n\n question: Name the women's doubles for werner schlager [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_28138035_4 (womens_doubles VARCHAR, mens_singles VARCHAR)\n\n question: Name the women's doubles for werner schlager [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE", # noqa: E501
58
- ]
59
- expected_lora_output = [
60
- " SELECT icao FROM table_name_74 WHERE airport = 'lilongwe international airport' ", # noqa: E501
61
- " SELECT nationality FROM table_name_11 WHERE elector = 'anchero pantaleone' ", # noqa: E501
62
- " SELECT one_mora FROM table_name_95 WHERE gloss = 'low tone mora with a gloss of /˩okiru/' [òkìɽɯ́] AND accented_mora = 'low tone mora with a gloss of /˩okiru/' [òkìɽɯ́] ", # noqa: E501
63
- " SELECT sex FROM people WHERE people_id IN (SELECT people_id FROM candidate GROUP BY sex ORDER BY COUNT(people_id) DESC LIMIT 1) ", # noqa: E501
64
- " SELECT pick FROM table_name_60 WHERE former_wnba_team = 'Minnesota Lynx' ", # noqa: E501
65
- " SELECT womens_doubles FROM table_28138035_4 WHERE mens_singles = 'Werner Schlager' " # noqa: E501
66
- ]
67
-
68
- print("lora adapter created")
69
- assert do_sample(llm, sql_lora_files, lora_id=0) == expected_no_lora_output
70
-
71
- print("lora 1")
72
- assert do_sample(llm, sql_lora_files, lora_id=1) == expected_lora_output
73
-
74
- print("no lora")
75
- assert do_sample(llm, sql_lora_files, lora_id=0) == expected_no_lora_output
76
-
77
- print("lora 2")
78
- assert do_sample(llm, sql_lora_files, lora_id=2) == expected_lora_output
79
-
80
- print("removing lora")
81
-
82
-
83
-def test_llama_tensor_parallel_equality(sql_lora_files, num_gpus_available):
84
- if num_gpus_available < 4:
85
- pytest.skip("Not enough GPUs for tensor parallelism 4")
86
-
87
- llm_tp1 = vllm.LLM(MODEL_PATH,
88
- enable_lora=True,
89
- max_num_seqs=16,
90
- max_loras=4,
91
- tensor_parallel_size=1)
92
- output_tp1 = do_sample(llm_tp1, sql_lora_files, lora_id=1)
93
-
94
- del llm_tp1
95
- cleanup_dist_env_and_memory()
96
-
97
- llm_tp2 = vllm.LLM(MODEL_PATH,
98
- enable_lora=True,
99
- max_num_seqs=16,
100
- max_loras=4,
101
- tensor_parallel_size=2)
102
- output_tp2 = do_sample(llm_tp2, sql_lora_files, lora_id=1)
103
-
104
- del llm_tp2
105
- cleanup_dist_env_and_memory()
106
-
107
- assert output_tp1 == output_tp2
108
-
109
- llm_tp4 = vllm.LLM(MODEL_PATH,
110
- enable_lora=True,
111
- max_num_seqs=16,
112
- max_loras=4,
113
- tensor_parallel_size=4)
114
- output_tp4 = do_sample(llm_tp4, sql_lora_files, lora_id=1)
115
-
116
- del llm_tp4
117
- cleanup_dist_env_and_memory()
118
-
119
- assert output_tp1 == output_tp4
120
-
121
-
122
-def test_llama_lora_warmup(sql_lora_files):
123
- """Test that the LLM initialization works with a warmup LORA path and
124
- is more conservative"""
125
-
126
- @ray.remote(num_gpus=1)
127
- def get_num_gpu_blocks_lora():
128
- llm = vllm.LLM(MODEL_PATH, enable_lora=True, max_num_seqs=16)
129
- num_gpu_blocks_lora_warmup = llm.llm_engine.cache_config.num_gpu_blocks
130
- return num_gpu_blocks_lora_warmup
131
-
132
- @ray.remote(num_gpus=1)
133
- def get_num_gpu_blocks_no_lora():
134
- llm = vllm.LLM(MODEL_PATH, max_num_seqs=16)
135
- num_gpu_blocks_no_lora_warmup = (
136
- llm.llm_engine.cache_config.num_gpu_blocks)
137
- return num_gpu_blocks_no_lora_warmup
138
-
139
- num_gpu_blocks_lora_warmup = ray.get(get_num_gpu_blocks_lora.remote())
140
- num_gpu_blocks_no_lora_warmup = ray.get(
141
- get_num_gpu_blocks_no_lora.remote())
142
- assert num_gpu_blocks_lora_warmup < num_gpu_blocks_no_lora_warmup, (
143
- "The warmup with lora should be more "
144
- "conservative than without lora, therefore the number of "
145
- "memory blocks for the KV cache should be "
146
- "less when using lora than when not using lora")
tests/lora/test_llama_tp.py
new
+161
@@ -0,0 +1,161 @@
1
+from typing import List
2
+
3
+import ray
4
+
5
+import vllm
6
+from tests.utils import fork_new_process_for_each_test
7
+from vllm.lora.request import LoRARequest
8
+
9
+from ..utils import multi_gpu_test
10
+
11
+MODEL_PATH = "meta-llama/Llama-2-7b-hf"
12
+
13
+EXPECTED_NO_LORA_OUTPUT = [
14
+ "\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_75 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_76 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_77 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_78 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user]", # noqa: E501
15
+ " Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_11 (nationality VARCHAR, elector VARCHAR)\n\n question: When Anchero Pantaleone was the elector what is under nationality? ", # noqa: E501
16
+ "\n\n answer: 1\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_96 (one_mora VARCHAR, gloss VARCHAR, accented_mora VARCHAR)\n\n question: What is the one mora for a high tone mora with a gloss of /˧kot/ [kòt]? [/user] [assistant]\n\n answer: 2\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_97 (one_mora VARCHAR, gloss VARCHAR, accented_mora VARCHAR)\n\n question: What is the one mora for a high tone mora with a gloss of /˧kot/ [kòt]? [/user] [assistant]\n\n answer: 2\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_98 (one_mora VARCHAR, gloss VARCHAR, accented_mora VARCHAR)\n\n question: What is the one m", # noqa: E501
17
+ " Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE candidate (people_id VARCHAR, unsure_rate INTEGER); CREATE TABLE people (sex VARCHAR, people_id VARCHAR)\n\n question: which gender got the highest average uncertain ratio. ", # noqa: E501
18
+ " Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_60 (pick INTEGER, former_wnba_team VARCHAR)\n\n question: What pick was a player that previously played for the Minnesota Lynx? ", # noqa: E501
19
+ "\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_28138035_4 (womens_doubles VARCHAR, mens_singles VARCHAR)\n\n question: Name the women's doubles for werner schlager [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_28138035_4 (womens_doubles VARCHAR, mens_singles VARCHAR)\n\n question: Name the women's doubles for werner schlager [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_28138035_4 (womens_doubles VARCHAR, mens_singles VARCHAR)\n\n question: Name the women's doubles for werner schlager [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE", # noqa: E501
20
+]
21
+EXPECTED_LORA_OUTPUT = [
22
+ " SELECT icao FROM table_name_74 WHERE airport = 'lilongwe international airport' ", # noqa: E501
23
+ " SELECT nationality FROM table_name_11 WHERE elector = 'anchero pantaleone' ", # noqa: E501
24
+ " SELECT one_mora FROM table_name_95 WHERE gloss = 'low tone mora with a gloss of /˩okiru/' [òkìɽɯ́] AND accented_mora = 'low tone mora with a gloss of /˩okiru/' [òkìɽɯ́] ", # noqa: E501
25
+ " SELECT sex FROM people WHERE people_id IN (SELECT people_id FROM candidate GROUP BY sex ORDER BY COUNT(people_id) DESC LIMIT 1) ", # noqa: E501
26
+ " SELECT pick FROM table_name_60 WHERE former_wnba_team = 'Minnesota Lynx' ", # noqa: E501
27
+ " SELECT womens_doubles FROM table_28138035_4 WHERE mens_singles = 'Werner Schlager' " # noqa: E501
28
+]
29
+
30
+
31
+def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:
32
+ prompts = [
33
+ "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_74 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]", # noqa: E501
34
+ "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_11 (nationality VARCHAR, elector VARCHAR)\n\n question: When Anchero Pantaleone was the elector what is under nationality? [/user] [assistant]", # noqa: E501
35
+ "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_95 (one_mora VARCHAR, gloss VARCHAR, accented_mora VARCHAR)\n\n question: What is the one mora for a low tone mora with a gloss of /˩okiru/ [òkìɽɯ́]? [/user] [assistant]", # noqa: E501
36
+ "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE candidate (people_id VARCHAR, unsure_rate INTEGER); CREATE TABLE people (sex VARCHAR, people_id VARCHAR)\n\n question: which gender got the highest average uncertain ratio. [/user] [assistant]", # noqa: E501
37
+ "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_60 (pick INTEGER, former_wnba_team VARCHAR)\n\n question: What pick was a player that previously played for the Minnesota Lynx? [/user] [assistant]", # noqa: E501
38
+ "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_28138035_4 (womens_doubles VARCHAR, mens_singles VARCHAR)\n\n question: Name the women's doubles for werner schlager [/user] [assistant]" # noqa: E501
39
+ ]
40
+ sampling_params = vllm.SamplingParams(temperature=0,
41
+ max_tokens=256,
42
+ stop=["[/assistant]"])
43
+ outputs = llm.generate(
44
+ prompts,
45
+ sampling_params,
46
+ lora_request=LoRARequest(str(lora_id), lora_id, lora_path)
47
+ if lora_id else None)
48
+ # Print the outputs.
49
+ generated_texts: List[str] = []
50
+ for output in outputs:
51
+ prompt = output.prompt
52
+ generated_text = output.outputs[0].text
53
+ generated_texts.append(generated_text)
54
+ print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
55
+ return generated_texts
56
+
57
+
58
+@fork_new_process_for_each_test
59
+def test_llama_lora(sql_lora_files):
60
+
61
+ llm = vllm.LLM(MODEL_PATH,
62
+ enable_lora=True,
63
+ max_num_seqs=16,
64
+ max_loras=4,
65
+ tensor_parallel_size=1)
66
+
67
+ print("lora adapter created")
68
+ assert do_sample(llm, sql_lora_files, lora_id=0) == EXPECTED_NO_LORA_OUTPUT
69
+
70
+ print("lora 1")
71
+ assert do_sample(llm, sql_lora_files, lora_id=1) == EXPECTED_LORA_OUTPUT
72
+
73
+ print("no lora")
74
+ assert do_sample(llm, sql_lora_files, lora_id=0) == EXPECTED_NO_LORA_OUTPUT
75
+
76
+ print("lora 2")
77
+ assert do_sample(llm, sql_lora_files, lora_id=2) == EXPECTED_LORA_OUTPUT
78
+
79
+ print("removing lora")
80
+
81
+
82
+@fork_new_process_for_each_test
83
+def test_llama_lora_warmup(sql_lora_files):
84
+ """Test that the LLM initialization works with a warmup LORA path and
85
+ is more conservative"""
86
+
87
+ @ray.remote(num_gpus=1)
88
+ def get_num_gpu_blocks_lora():
89
+ llm = vllm.LLM(MODEL_PATH, enable_lora=True, max_num_seqs=16)
90
+ num_gpu_blocks_lora_warmup = llm.llm_engine.cache_config.num_gpu_blocks
91
+ return num_gpu_blocks_lora_warmup
92
+
93
+ @ray.remote(num_gpus=1)
94
+ def get_num_gpu_blocks_no_lora():
95
+ llm = vllm.LLM(MODEL_PATH, max_num_seqs=16)
96
+ num_gpu_blocks_no_lora_warmup = (
97
+ llm.llm_engine.cache_config.num_gpu_blocks)
98
+ return num_gpu_blocks_no_lora_warmup
99
+
100
+ num_gpu_blocks_lora_warmup = ray.get(get_num_gpu_blocks_lora.remote())
101
+ num_gpu_blocks_no_lora_warmup = ray.get(
102
+ get_num_gpu_blocks_no_lora.remote())
103
+ assert num_gpu_blocks_lora_warmup < num_gpu_blocks_no_lora_warmup, (
104
+ "The warmup with lora should be more "
105
+ "conservative than without lora, therefore the number of "
106
+ "memory blocks for the KV cache should be "
107
+ "less when using lora than when not using lora")
108
+
109
+
110
+@multi_gpu_test(num_gpus=4)
111
+@fork_new_process_for_each_test
112
+def test_llama_lora_tp4(sql_lora_files):
113
+
114
+ llm = vllm.LLM(
115
+ MODEL_PATH,
116
+ enable_lora=True,
117
+ max_num_seqs=16,
118
+ max_loras=4,
119
+ tensor_parallel_size=4,
120
+ )
121
+
122
+ print("lora adapter created")
123
+ assert do_sample(llm, sql_lora_files, lora_id=0) == EXPECTED_NO_LORA_OUTPUT
124
+
125
+ print("lora 1")
126
+ assert do_sample(llm, sql_lora_files, lora_id=1) == EXPECTED_LORA_OUTPUT
127
+
128
+ print("no lora")
129
+ assert do_sample(llm, sql_lora_files, lora_id=0) == EXPECTED_NO_LORA_OUTPUT
130
+
131
+ print("lora 2")
132
+ assert do_sample(llm, sql_lora_files, lora_id=2) == EXPECTED_LORA_OUTPUT
133
+
134
+ print("removing lora")
135
+
136
+
137
+@multi_gpu_test(num_gpus=4)
138
+@fork_new_process_for_each_test
139
+def test_llama_lora_tp4_fully_sharded_loras(sql_lora_files):
140
+
141
+ llm = vllm.LLM(
142
+ MODEL_PATH,
143
+ enable_lora=True,
144
+ max_num_seqs=16,
145
+ max_loras=4,
146
+ tensor_parallel_size=4,
147
+ fully_sharded_loras=True,
148
+ )
149
+ print("lora adapter created")
150
+ assert do_sample(llm, sql_lora_files, lora_id=0) == EXPECTED_NO_LORA_OUTPUT
151
+
152
+ print("lora 1")
153
+ assert do_sample(llm, sql_lora_files, lora_id=1) == EXPECTED_LORA_OUTPUT
154
+
155
+ print("no lora")
156
+ assert do_sample(llm, sql_lora_files, lora_id=0) == EXPECTED_NO_LORA_OUTPUT
157
+
158
+ print("lora 2")
159
+ assert do_sample(llm, sql_lora_files, lora_id=2) == EXPECTED_LORA_OUTPUT
160
+
161
+ print("removing lora")
vllm/lora/fully_sharded_layers.py
+5
@@ -44,6 +44,11 @@ class ColumnParallelLinearWithShardedLoRA(ColumnParallelLinearWithLoRA):
44
Based on S-LoRA, slicing happens along the rank dim.
45
"""
46
47
+ # For all LoRA layers where the `base_layer` is `ColumnParallelLinear`,
48
+ # their `lora_a` and `lora_b` have different sharding patterns. After
49
+ # completing the `lora_a` GEMM , a gather operation is performed.
50
+ # Therefore, the sharding of `lora_a` only needs to correspond with the
51
+ # gather operation.
52
def slice_lora_a(self, lora_a: torch.Tensor) -> torch.Tensor:
53
tp_rank = get_tensor_model_parallel_rank()
54
shard_size = self.lora_a_stacked.shape[2]
vllm/lora/layers.py
+28
-6
@@ -451,6 +451,12 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
451
452
def __init__(self, base_layer: ColumnParallelLinear) -> None:
453
super().__init__()
454
+ # The base_layer type is ColumnParallelLinear or
455
+ # MergedColumnParallelLinear, their weight sharding logic is
456
+ # inconsistent when TP is greater than 1.
457
+ self.is_merged_col_linear = type(
458
+ base_layer) is MergedColumnParallelLinear
459
+
460
self.base_layer = base_layer
461
self.tp_size = get_tensor_model_parallel_world_size()
462
self.input_size = self.base_layer.input_size
@@ -508,14 +514,30 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
514
return lora_a
515
516
def slice_lora_b(self, lora_b: torch.Tensor) -> torch.Tensor:
511
- tensor_model_parallel_rank = get_tensor_model_parallel_rank()
512
- shard_size = self.output_dim
513
- start_idx = tensor_model_parallel_rank * shard_size
514
- end_idx = (tensor_model_parallel_rank + 1) * shard_size
515
- lora_b = lora_b[:, start_idx:end_idx]
517
+ # Applicable to cases where the base_layer is
518
+ # MergedColumnParallelLinear.
519
+ if self.is_merged_col_linear:
520
+ tp_rank = get_tensor_model_parallel_rank()
521
+ shard_size = self.output_size // 2
522
+ offset = lora_b.shape[-1] // 2
523
+
524
+ left_weight = lora_b[:, tp_rank * shard_size:(tp_rank + 1) *
525
+ shard_size]
526
+ right_weight = lora_b[:, offset + tp_rank * shard_size:offset +
527
+ (tp_rank + 1) * shard_size]
528
+ lora_b = torch.cat([left_weight, right_weight], dim=1)
529
+ # Applicable to cases where the base_layer is
530
+ # ColumnParallelLinear.
531
+ else:
532
+ tensor_model_parallel_rank = get_tensor_model_parallel_rank()
533
+ shard_size = self.output_dim
534
+ start_idx = tensor_model_parallel_rank * shard_size
535
+ end_idx = (tensor_model_parallel_rank + 1) * shard_size
536
+ lora_b = lora_b[:, start_idx:end_idx]
537
return lora_b
538
539
def slice_bias(self, bias: torch.Tensor) -> torch.Tensor:
540
+ # TODO: Fix the slicing logic of bias.
541
if bias is None:
542
return bias
543
tensor_model_parallel_rank = get_tensor_model_parallel_rank()
@@ -779,7 +801,7 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
801
class QKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
802
"""
803
ColumnParallelLinear layer that is specifically designed for
782
- qkv_proj. Certain models, such as chtglm3 and baichuan-7b,
804
+ qkv_proj. Certain models, such as chatglm3 and baichuan-7b,
805
only contains a single LoRA within their qkv_proj layer.
806
807
During inference with Tensor Parallel, the weights of lora_b
vllm/model_executor/models/chatglm.py
+2
-2
@@ -760,7 +760,7 @@ class ChatGLMForCausalLM(ChatGLMBaseModel, SupportsLoRA, SupportsPP,
760
config = vllm_config.model_config.hf_config
761
# Initialize VL
762
if hasattr(config, "visual"):
763
- return ChatGLM(vllm_config=vllm_config, prefix=prefix)
763
+ return ChatGLMV(vllm_config=vllm_config, prefix=prefix)
764
# Initialize LLM
765
else:
766
- return ChatGLMV(vllm_config=vllm_config, prefix=prefix)
766
+ return ChatGLM(vllm_config=vllm_config, prefix=prefix)