@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1700c543a5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[Bugfix] Fix LoRA weight sharding (#10450)

Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>

Jee Jee Li committed Nov 24, 2024 at 09:23 UTC 1700c543a556e669e559c369a36c0a0d36a8de19
7 files changed +258 -168
.buildkite/test-pipeline.yaml
+9 -4
@@ -230,7 +230,7 @@ steps:
230 source_file_dependencies:
231 - vllm/lora
232 - tests/lora
233 - command: pytest -v -s lora --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --ignore=lora/test_long_context.py
233 + command: pytest -v -s lora --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --ignore lora/test_long_context.py lora/test_chatglm3_tp.py lora/test_llama_tp.py
234 parallelism: 4
235
236 - label: "PyTorch Fullgraph Smoke Test" # 9min
@@ -475,18 +475,23 @@ steps:
475 - pytest -v -s distributed/test_pp_cudagraph.py
476 - pytest -v -s distributed/test_pipeline_parallel.py
477
478 -- label: LoRA Long Context (Distributed) # 11min
479 - # This test runs llama 13B, so it is required to run on 4 GPUs.
478 +- label: LoRA TP Test (Distributed)
479 num_gpus: 4
480 soft_fail: true
481 source_file_dependencies:
482 - vllm/lora
484 - - tests/lora/test_long_context
483 + - tests/lora
484 commands:
485 # FIXIT: find out which code initialize cuda before running the test
486 # before the fix, we need to use spawn to test it
487 - export VLLM_WORKER_MULTIPROC_METHOD=spawn
488 + # This test runs llama 13B, so it is required to run on 4 GPUs.
489 - pytest -v -s -x lora/test_long_context.py
490 + # There is some Tensor Parallelism related processing logic in LoRA that
491 + # requires multi-GPU testing for validation.
492 + - pytest -v -s -x lora/test_chatglm3_tp.py
493 + - pytest -v -s -x lora/test_llama_tp.py
494 +
495
496 - label: Weight Loading Multiple GPU Test # 33min
497 working_dir: "/vllm-workspace/tests"
tests/lora/test_chatglm3_tp.py renamed
+53 -10
@@ -1,12 +1,21 @@
1 from typing import List
2
3 import vllm
4 +from tests.utils import fork_new_process_for_each_test
5 from vllm.lora.request import LoRARequest
6
7 +from ..utils import multi_gpu_test
8 +
9 MODEL_PATH = "THUDM/chatglm3-6b"
10
11 PROMPT_TEMPLATE = """I want you to act as a SQL terminal in front of an example database, you need only to return the sql command to me.Below is an instruction that describes a task, Write a response that appropriately completes the request.\n"\n##Instruction:\nconcert_singer contains tables such as stadium, singer, concert, singer_in_concert. Table stadium has columns such as Stadium_ID, Location, Name, Capacity, Highest, Lowest, Average. Stadium_ID is the primary key.\nTable singer has columns such as Singer_ID, Name, Country, Song_Name, Song_release_year, Age, Is_male. Singer_ID is the primary key.\nTable concert has columns such as concert_ID, concert_Name, Theme, Stadium_ID, Year. concert_ID is the primary key.\nTable singer_in_concert has columns such as concert_ID, Singer_ID. concert_ID is the primary key.\nThe Stadium_ID of concert is the foreign key of Stadium_ID of stadium.\nThe Singer_ID of singer_in_concert is the foreign key of Singer_ID of singer.\nThe concert_ID of singer_in_concert is the foreign key of concert_ID of concert.\n\n###Input:\n{query}\n\n###Response:""" # noqa: E501
12
13 +EXPECTED_LORA_OUTPUT = [
14 + "SELECT count(*) FROM singer",
15 + "SELECT avg(age) , min(age) , max(age) FROM singer WHERE country = 'France'", # noqa: E501
16 + "SELECT name , country , age FROM singer ORDER BY age",
17 +]
18 +
19
20 def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:
21 prompts = [
@@ -20,7 +29,6 @@ def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:
29 "Show name, country, age for all singers ordered by age from the oldest to the youngest." # noqa: E501
30 ),
31 ]
23 - print(prompts)
32 sampling_params = vllm.SamplingParams(temperature=0, max_tokens=32)
33 outputs = llm.generate(
34 prompts,
@@ -37,23 +45,58 @@ def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:
45 return generated_texts
46
47
48 +@fork_new_process_for_each_test
49 def test_chatglm3_lora(chatglm3_lora_files):
50 llm = vllm.LLM(MODEL_PATH,
51 max_model_len=1024,
52 enable_lora=True,
53 max_loras=4,
54 max_lora_rank=64,
55 + tensor_parallel_size=1,
56 trust_remote_code=True)
57
48 - expected_lora_output = [
49 - "SELECT count(*) FROM singer",
50 - "SELECT avg(age) , min(age) , max(age) FROM singer WHERE country = 'France'", # noqa: E501
51 - "SELECT name , country , age FROM singer ORDER BY age",
52 - ]
58 + output1 = do_sample(llm, chatglm3_lora_files, lora_id=1)
59 + for i in range(len(EXPECTED_LORA_OUTPUT)):
60 + assert output1[i] == EXPECTED_LORA_OUTPUT[i]
61 + output2 = do_sample(llm, chatglm3_lora_files, lora_id=2)
62 + for i in range(len(EXPECTED_LORA_OUTPUT)):
63 + assert output2[i] == EXPECTED_LORA_OUTPUT[i]
64 +
65
66 +@multi_gpu_test(num_gpus=4)
67 +@fork_new_process_for_each_test
68 +def test_chatglm3_lora_tp4(chatglm3_lora_files):
69 + llm = vllm.LLM(MODEL_PATH,
70 + max_model_len=1024,
71 + enable_lora=True,
72 + max_loras=4,
73 + max_lora_rank=64,
74 + tensor_parallel_size=4,
75 + trust_remote_code=True,
76 + fully_sharded_loras=False)
77 +
78 + output1 = do_sample(llm, chatglm3_lora_files, lora_id=1)
79 + for i in range(len(EXPECTED_LORA_OUTPUT)):
80 + assert output1[i] == EXPECTED_LORA_OUTPUT[i]
81 + output2 = do_sample(llm, chatglm3_lora_files, lora_id=2)
82 + for i in range(len(EXPECTED_LORA_OUTPUT)):
83 + assert output2[i] == EXPECTED_LORA_OUTPUT[i]
84 +
85 +
86 +@multi_gpu_test(num_gpus=4)
87 +@fork_new_process_for_each_test
88 +def test_chatglm3_lora_tp4_fully_sharded_loras(chatglm3_lora_files):
89 + llm = vllm.LLM(MODEL_PATH,
90 + max_model_len=1024,
91 + enable_lora=True,
92 + max_loras=4,
93 + max_lora_rank=64,
94 + tensor_parallel_size=4,
95 + trust_remote_code=True,
96 + fully_sharded_loras=True)
97 output1 = do_sample(llm, chatglm3_lora_files, lora_id=1)
55 - for i in range(len(expected_lora_output)):
56 - assert output1[i] == expected_lora_output[i]
98 + for i in range(len(EXPECTED_LORA_OUTPUT)):
99 + assert output1[i] == EXPECTED_LORA_OUTPUT[i]
100 output2 = do_sample(llm, chatglm3_lora_files, lora_id=2)
58 - for i in range(len(expected_lora_output)):
59 - assert output2[i] == expected_lora_output[i]
101 + for i in range(len(EXPECTED_LORA_OUTPUT)):
102 + assert output2[i] == EXPECTED_LORA_OUTPUT[i]
tests/lora/test_llama.py deleted
-146
@@ -1,146 +0,0 @@
1 -from typing import List
2 -
3 -import pytest
4 -import ray
5 -
6 -import vllm
7 -from vllm.distributed import cleanup_dist_env_and_memory
8 -from vllm.lora.request import LoRARequest
9 -
10 -MODEL_PATH = "meta-llama/Llama-2-7b-hf"
11 -
12 -
13 -def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:
14 - prompts = [
15 - "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_74 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]", # noqa: E501
16 - "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_11 (nationality VARCHAR, elector VARCHAR)\n\n question: When Anchero Pantaleone was the elector what is under nationality? [/user] [assistant]", # noqa: E501
17 - "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_95 (one_mora VARCHAR, gloss VARCHAR, accented_mora VARCHAR)\n\n question: What is the one mora for a low tone mora with a gloss of /˩okiru/ [òkìɽɯ́]? [/user] [assistant]", # noqa: E501
18 - "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE candidate (people_id VARCHAR, unsure_rate INTEGER); CREATE TABLE people (sex VARCHAR, people_id VARCHAR)\n\n question: which gender got the highest average uncertain ratio. [/user] [assistant]", # noqa: E501
19 - "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_60 (pick INTEGER, former_wnba_team VARCHAR)\n\n question: What pick was a player that previously played for the Minnesota Lynx? [/user] [assistant]", # noqa: E501
20 - "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_28138035_4 (womens_doubles VARCHAR, mens_singles VARCHAR)\n\n question: Name the women's doubles for werner schlager [/user] [assistant]" # noqa: E501
21 - ]
22 - sampling_params = vllm.SamplingParams(temperature=0,
23 - max_tokens=256,
24 - stop=["[/assistant]"])
25 - outputs = llm.generate(
26 - prompts,
27 - sampling_params,
28 - lora_request=LoRARequest(str(lora_id), lora_id, lora_path)
29 - if lora_id else None)
30 - # Print the outputs.
31 - generated_texts: List[str] = []
32 - for output in outputs:
33 - prompt = output.prompt
34 - generated_text = output.outputs[0].text
35 - generated_texts.append(generated_text)
36 - print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
37 - return generated_texts
38 -
39 -
40 -@pytest.mark.parametrize("tp_size", [1, 2, 4])
41 -def test_llama_lora(sql_lora_files, tp_size, num_gpus_available):
42 - if num_gpus_available < tp_size:
43 - pytest.skip(f"Not enough GPUs for tensor parallelism {tp_size}")
44 -
45 - llm = vllm.LLM(MODEL_PATH,
46 - enable_lora=True,
47 - max_num_seqs=16,
48 - max_loras=4,
49 - tensor_parallel_size=tp_size)
50 -
51 - expected_no_lora_output = [
52 - "\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_75 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_76 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_77 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_78 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user]", # noqa: E501
53 - " Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_11 (nationality VARCHAR, elector VARCHAR)\n\n question: When Anchero Pantaleone was the elector what is under nationality? ", # noqa: E501
54 - "\n\n answer: 1\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_96 (one_mora VARCHAR, gloss VARCHAR, accented_mora VARCHAR)\n\n question: What is the one mora for a high tone mora with a gloss of /˧kot/ [kòt]? [/user] [assistant]\n\n answer: 2\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_97 (one_mora VARCHAR, gloss VARCHAR, accented_mora VARCHAR)\n\n question: What is the one mora for a high tone mora with a gloss of /˧kot/ [kòt]? [/user] [assistant]\n\n answer: 2\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_98 (one_mora VARCHAR, gloss VARCHAR, accented_mora VARCHAR)\n\n question: What is the one m", # noqa: E501
55 - " Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE candidate (people_id VARCHAR, unsure_rate INTEGER); CREATE TABLE people (sex VARCHAR, people_id VARCHAR)\n\n question: which gender got the highest average uncertain ratio. ", # noqa: E501
56 - " Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_60 (pick INTEGER, former_wnba_team VARCHAR)\n\n question: What pick was a player that previously played for the Minnesota Lynx? ", # noqa: E501
57 - "\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_28138035_4 (womens_doubles VARCHAR, mens_singles VARCHAR)\n\n question: Name the women's doubles for werner schlager [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_28138035_4 (womens_doubles VARCHAR, mens_singles VARCHAR)\n\n question: Name the women's doubles for werner schlager [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_28138035_4 (womens_doubles VARCHAR, mens_singles VARCHAR)\n\n question: Name the women's doubles for werner schlager [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE", # noqa: E501
58 - ]
59 - expected_lora_output = [
60 - " SELECT icao FROM table_name_74 WHERE airport = 'lilongwe international airport' ", # noqa: E501
61 - " SELECT nationality FROM table_name_11 WHERE elector = 'anchero pantaleone' ", # noqa: E501
62 - " SELECT one_mora FROM table_name_95 WHERE gloss = 'low tone mora with a gloss of /˩okiru/' [òkìɽɯ́] AND accented_mora = 'low tone mora with a gloss of /˩okiru/' [òkìɽɯ́] ", # noqa: E501
63 - " SELECT sex FROM people WHERE people_id IN (SELECT people_id FROM candidate GROUP BY sex ORDER BY COUNT(people_id) DESC LIMIT 1) ", # noqa: E501
64 - " SELECT pick FROM table_name_60 WHERE former_wnba_team = 'Minnesota Lynx' ", # noqa: E501
65 - " SELECT womens_doubles FROM table_28138035_4 WHERE mens_singles = 'Werner Schlager' " # noqa: E501
66 - ]
67 -
68 - print("lora adapter created")
69 - assert do_sample(llm, sql_lora_files, lora_id=0) == expected_no_lora_output
70 -
71 - print("lora 1")
72 - assert do_sample(llm, sql_lora_files, lora_id=1) == expected_lora_output
73 -
74 - print("no lora")
75 - assert do_sample(llm, sql_lora_files, lora_id=0) == expected_no_lora_output
76 -
77 - print("lora 2")
78 - assert do_sample(llm, sql_lora_files, lora_id=2) == expected_lora_output
79 -
80 - print("removing lora")
81 -
82 -
83 -def test_llama_tensor_parallel_equality(sql_lora_files, num_gpus_available):
84 - if num_gpus_available < 4:
85 - pytest.skip("Not enough GPUs for tensor parallelism 4")
86 -
87 - llm_tp1 = vllm.LLM(MODEL_PATH,
88 - enable_lora=True,
89 - max_num_seqs=16,
90 - max_loras=4,
91 - tensor_parallel_size=1)
92 - output_tp1 = do_sample(llm_tp1, sql_lora_files, lora_id=1)
93 -
94 - del llm_tp1
95 - cleanup_dist_env_and_memory()
96 -
97 - llm_tp2 = vllm.LLM(MODEL_PATH,
98 - enable_lora=True,
99 - max_num_seqs=16,
100 - max_loras=4,
101 - tensor_parallel_size=2)
102 - output_tp2 = do_sample(llm_tp2, sql_lora_files, lora_id=1)
103 -
104 - del llm_tp2
105 - cleanup_dist_env_and_memory()
106 -
107 - assert output_tp1 == output_tp2
108 -
109 - llm_tp4 = vllm.LLM(MODEL_PATH,
110 - enable_lora=True,
111 - max_num_seqs=16,
112 - max_loras=4,
113 - tensor_parallel_size=4)
114 - output_tp4 = do_sample(llm_tp4, sql_lora_files, lora_id=1)
115 -
116 - del llm_tp4
117 - cleanup_dist_env_and_memory()
118 -
119 - assert output_tp1 == output_tp4
120 -
121 -
122 -def test_llama_lora_warmup(sql_lora_files):
123 - """Test that the LLM initialization works with a warmup LORA path and
124 - is more conservative"""
125 -
126 - @ray.remote(num_gpus=1)
127 - def get_num_gpu_blocks_lora():
128 - llm = vllm.LLM(MODEL_PATH, enable_lora=True, max_num_seqs=16)
129 - num_gpu_blocks_lora_warmup = llm.llm_engine.cache_config.num_gpu_blocks
130 - return num_gpu_blocks_lora_warmup
131 -
132 - @ray.remote(num_gpus=1)
133 - def get_num_gpu_blocks_no_lora():
134 - llm = vllm.LLM(MODEL_PATH, max_num_seqs=16)
135 - num_gpu_blocks_no_lora_warmup = (
136 - llm.llm_engine.cache_config.num_gpu_blocks)
137 - return num_gpu_blocks_no_lora_warmup
138 -
139 - num_gpu_blocks_lora_warmup = ray.get(get_num_gpu_blocks_lora.remote())
140 - num_gpu_blocks_no_lora_warmup = ray.get(
141 - get_num_gpu_blocks_no_lora.remote())
142 - assert num_gpu_blocks_lora_warmup < num_gpu_blocks_no_lora_warmup, (
143 - "The warmup with lora should be more "
144 - "conservative than without lora, therefore the number of "
145 - "memory blocks for the KV cache should be "
146 - "less when using lora than when not using lora")
tests/lora/test_llama_tp.py new
+161
@@ -0,0 +1,161 @@
1 +from typing import List
2 +
3 +import ray
4 +
5 +import vllm
6 +from tests.utils import fork_new_process_for_each_test
7 +from vllm.lora.request import LoRARequest
8 +
9 +from ..utils import multi_gpu_test
10 +
11 +MODEL_PATH = "meta-llama/Llama-2-7b-hf"
12 +
13 +EXPECTED_NO_LORA_OUTPUT = [
14 + "\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_75 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_76 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_77 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_78 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user]", # noqa: E501
15 + " Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_11 (nationality VARCHAR, elector VARCHAR)\n\n question: When Anchero Pantaleone was the elector what is under nationality? ", # noqa: E501
16 + "\n\n answer: 1\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_96 (one_mora VARCHAR, gloss VARCHAR, accented_mora VARCHAR)\n\n question: What is the one mora for a high tone mora with a gloss of /˧kot/ [kòt]? [/user] [assistant]\n\n answer: 2\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_97 (one_mora VARCHAR, gloss VARCHAR, accented_mora VARCHAR)\n\n question: What is the one mora for a high tone mora with a gloss of /˧kot/ [kòt]? [/user] [assistant]\n\n answer: 2\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_98 (one_mora VARCHAR, gloss VARCHAR, accented_mora VARCHAR)\n\n question: What is the one m", # noqa: E501
17 + " Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE candidate (people_id VARCHAR, unsure_rate INTEGER); CREATE TABLE people (sex VARCHAR, people_id VARCHAR)\n\n question: which gender got the highest average uncertain ratio. ", # noqa: E501
18 + " Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_60 (pick INTEGER, former_wnba_team VARCHAR)\n\n question: What pick was a player that previously played for the Minnesota Lynx? ", # noqa: E501
19 + "\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_28138035_4 (womens_doubles VARCHAR, mens_singles VARCHAR)\n\n question: Name the women's doubles for werner schlager [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_28138035_4 (womens_doubles VARCHAR, mens_singles VARCHAR)\n\n question: Name the women's doubles for werner schlager [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_28138035_4 (womens_doubles VARCHAR, mens_singles VARCHAR)\n\n question: Name the women's doubles for werner schlager [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE", # noqa: E501
20 +]
21 +EXPECTED_LORA_OUTPUT = [
22 + " SELECT icao FROM table_name_74 WHERE airport = 'lilongwe international airport' ", # noqa: E501
23 + " SELECT nationality FROM table_name_11 WHERE elector = 'anchero pantaleone' ", # noqa: E501
24 + " SELECT one_mora FROM table_name_95 WHERE gloss = 'low tone mora with a gloss of /˩okiru/' [òkìɽɯ́] AND accented_mora = 'low tone mora with a gloss of /˩okiru/' [òkìɽɯ́] ", # noqa: E501
25 + " SELECT sex FROM people WHERE people_id IN (SELECT people_id FROM candidate GROUP BY sex ORDER BY COUNT(people_id) DESC LIMIT 1) ", # noqa: E501
26 + " SELECT pick FROM table_name_60 WHERE former_wnba_team = 'Minnesota Lynx' ", # noqa: E501
27 + " SELECT womens_doubles FROM table_28138035_4 WHERE mens_singles = 'Werner Schlager' " # noqa: E501
28 +]
29 +
30 +
31 +def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:
32 + prompts = [
33 + "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_74 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]", # noqa: E501
34 + "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_11 (nationality VARCHAR, elector VARCHAR)\n\n question: When Anchero Pantaleone was the elector what is under nationality? [/user] [assistant]", # noqa: E501
35 + "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_95 (one_mora VARCHAR, gloss VARCHAR, accented_mora VARCHAR)\n\n question: What is the one mora for a low tone mora with a gloss of /˩okiru/ [òkìɽɯ́]? [/user] [assistant]", # noqa: E501
36 + "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE candidate (people_id VARCHAR, unsure_rate INTEGER); CREATE TABLE people (sex VARCHAR, people_id VARCHAR)\n\n question: which gender got the highest average uncertain ratio. [/user] [assistant]", # noqa: E501
37 + "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_60 (pick INTEGER, former_wnba_team VARCHAR)\n\n question: What pick was a player that previously played for the Minnesota Lynx? [/user] [assistant]", # noqa: E501
38 + "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_28138035_4 (womens_doubles VARCHAR, mens_singles VARCHAR)\n\n question: Name the women's doubles for werner schlager [/user] [assistant]" # noqa: E501
39 + ]
40 + sampling_params = vllm.SamplingParams(temperature=0,
41 + max_tokens=256,
42 + stop=["[/assistant]"])
43 + outputs = llm.generate(
44 + prompts,
45 + sampling_params,
46 + lora_request=LoRARequest(str(lora_id), lora_id, lora_path)
47 + if lora_id else None)
48 + # Print the outputs.
49 + generated_texts: List[str] = []
50 + for output in outputs:
51 + prompt = output.prompt
52 + generated_text = output.outputs[0].text
53 + generated_texts.append(generated_text)
54 + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
55 + return generated_texts
56 +
57 +
58 +@fork_new_process_for_each_test
59 +def test_llama_lora(sql_lora_files):
60 +
61 + llm = vllm.LLM(MODEL_PATH,
62 + enable_lora=True,
63 + max_num_seqs=16,
64 + max_loras=4,
65 + tensor_parallel_size=1)
66 +
67 + print("lora adapter created")
68 + assert do_sample(llm, sql_lora_files, lora_id=0) == EXPECTED_NO_LORA_OUTPUT
69 +
70 + print("lora 1")
71 + assert do_sample(llm, sql_lora_files, lora_id=1) == EXPECTED_LORA_OUTPUT
72 +
73 + print("no lora")
74 + assert do_sample(llm, sql_lora_files, lora_id=0) == EXPECTED_NO_LORA_OUTPUT
75 +
76 + print("lora 2")
77 + assert do_sample(llm, sql_lora_files, lora_id=2) == EXPECTED_LORA_OUTPUT
78 +
79 + print("removing lora")
80 +
81 +
82 +@fork_new_process_for_each_test
83 +def test_llama_lora_warmup(sql_lora_files):
84 + """Test that the LLM initialization works with a warmup LORA path and
85 + is more conservative"""
86 +
87 + @ray.remote(num_gpus=1)
88 + def get_num_gpu_blocks_lora():
89 + llm = vllm.LLM(MODEL_PATH, enable_lora=True, max_num_seqs=16)
90 + num_gpu_blocks_lora_warmup = llm.llm_engine.cache_config.num_gpu_blocks
91 + return num_gpu_blocks_lora_warmup
92 +
93 + @ray.remote(num_gpus=1)
94 + def get_num_gpu_blocks_no_lora():
95 + llm = vllm.LLM(MODEL_PATH, max_num_seqs=16)
96 + num_gpu_blocks_no_lora_warmup = (
97 + llm.llm_engine.cache_config.num_gpu_blocks)
98 + return num_gpu_blocks_no_lora_warmup
99 +
100 + num_gpu_blocks_lora_warmup = ray.get(get_num_gpu_blocks_lora.remote())
101 + num_gpu_blocks_no_lora_warmup = ray.get(
102 + get_num_gpu_blocks_no_lora.remote())
103 + assert num_gpu_blocks_lora_warmup < num_gpu_blocks_no_lora_warmup, (
104 + "The warmup with lora should be more "
105 + "conservative than without lora, therefore the number of "
106 + "memory blocks for the KV cache should be "
107 + "less when using lora than when not using lora")
108 +
109 +
110 +@multi_gpu_test(num_gpus=4)
111 +@fork_new_process_for_each_test
112 +def test_llama_lora_tp4(sql_lora_files):
113 +
114 + llm = vllm.LLM(
115 + MODEL_PATH,
116 + enable_lora=True,
117 + max_num_seqs=16,
118 + max_loras=4,
119 + tensor_parallel_size=4,
120 + )
121 +
122 + print("lora adapter created")
123 + assert do_sample(llm, sql_lora_files, lora_id=0) == EXPECTED_NO_LORA_OUTPUT
124 +
125 + print("lora 1")
126 + assert do_sample(llm, sql_lora_files, lora_id=1) == EXPECTED_LORA_OUTPUT
127 +
128 + print("no lora")
129 + assert do_sample(llm, sql_lora_files, lora_id=0) == EXPECTED_NO_LORA_OUTPUT
130 +
131 + print("lora 2")
132 + assert do_sample(llm, sql_lora_files, lora_id=2) == EXPECTED_LORA_OUTPUT
133 +
134 + print("removing lora")
135 +
136 +
137 +@multi_gpu_test(num_gpus=4)
138 +@fork_new_process_for_each_test
139 +def test_llama_lora_tp4_fully_sharded_loras(sql_lora_files):
140 +
141 + llm = vllm.LLM(
142 + MODEL_PATH,
143 + enable_lora=True,
144 + max_num_seqs=16,
145 + max_loras=4,
146 + tensor_parallel_size=4,
147 + fully_sharded_loras=True,
148 + )
149 + print("lora adapter created")
150 + assert do_sample(llm, sql_lora_files, lora_id=0) == EXPECTED_NO_LORA_OUTPUT
151 +
152 + print("lora 1")
153 + assert do_sample(llm, sql_lora_files, lora_id=1) == EXPECTED_LORA_OUTPUT
154 +
155 + print("no lora")
156 + assert do_sample(llm, sql_lora_files, lora_id=0) == EXPECTED_NO_LORA_OUTPUT
157 +
158 + print("lora 2")
159 + assert do_sample(llm, sql_lora_files, lora_id=2) == EXPECTED_LORA_OUTPUT
160 +
161 + print("removing lora")
vllm/lora/fully_sharded_layers.py
+5
@@ -44,6 +44,11 @@ class ColumnParallelLinearWithShardedLoRA(ColumnParallelLinearWithLoRA):
44 Based on S-LoRA, slicing happens along the rank dim.
45 """
46
47 + # For all LoRA layers where the `base_layer` is `ColumnParallelLinear`,
48 + # their `lora_a` and `lora_b` have different sharding patterns. After
49 + # completing the `lora_a` GEMM , a gather operation is performed.
50 + # Therefore, the sharding of `lora_a` only needs to correspond with the
51 + # gather operation.
52 def slice_lora_a(self, lora_a: torch.Tensor) -> torch.Tensor:
53 tp_rank = get_tensor_model_parallel_rank()
54 shard_size = self.lora_a_stacked.shape[2]
vllm/lora/layers.py
+28 -6
@@ -451,6 +451,12 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
451
452 def __init__(self, base_layer: ColumnParallelLinear) -> None:
453 super().__init__()
454 + # The base_layer type is ColumnParallelLinear or
455 + # MergedColumnParallelLinear, their weight sharding logic is
456 + # inconsistent when TP is greater than 1.
457 + self.is_merged_col_linear = type(
458 + base_layer) is MergedColumnParallelLinear
459 +
460 self.base_layer = base_layer
461 self.tp_size = get_tensor_model_parallel_world_size()
462 self.input_size = self.base_layer.input_size
@@ -508,14 +514,30 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
514 return lora_a
515
516 def slice_lora_b(self, lora_b: torch.Tensor) -> torch.Tensor:
511 - tensor_model_parallel_rank = get_tensor_model_parallel_rank()
512 - shard_size = self.output_dim
513 - start_idx = tensor_model_parallel_rank * shard_size
514 - end_idx = (tensor_model_parallel_rank + 1) * shard_size
515 - lora_b = lora_b[:, start_idx:end_idx]
517 + # Applicable to cases where the base_layer is
518 + # MergedColumnParallelLinear.
519 + if self.is_merged_col_linear:
520 + tp_rank = get_tensor_model_parallel_rank()
521 + shard_size = self.output_size // 2
522 + offset = lora_b.shape[-1] // 2
523 +
524 + left_weight = lora_b[:, tp_rank * shard_size:(tp_rank + 1) *
525 + shard_size]
526 + right_weight = lora_b[:, offset + tp_rank * shard_size:offset +
527 + (tp_rank + 1) * shard_size]
528 + lora_b = torch.cat([left_weight, right_weight], dim=1)
529 + # Applicable to cases where the base_layer is
530 + # ColumnParallelLinear.
531 + else:
532 + tensor_model_parallel_rank = get_tensor_model_parallel_rank()
533 + shard_size = self.output_dim
534 + start_idx = tensor_model_parallel_rank * shard_size
535 + end_idx = (tensor_model_parallel_rank + 1) * shard_size
536 + lora_b = lora_b[:, start_idx:end_idx]
537 return lora_b
538
539 def slice_bias(self, bias: torch.Tensor) -> torch.Tensor:
540 + # TODO: Fix the slicing logic of bias.
541 if bias is None:
542 return bias
543 tensor_model_parallel_rank = get_tensor_model_parallel_rank()
@@ -779,7 +801,7 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
801 class QKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
802 """
803 ColumnParallelLinear layer that is specifically designed for
782 - qkv_proj. Certain models, such as chtglm3 and baichuan-7b,
804 + qkv_proj. Certain models, such as chatglm3 and baichuan-7b,
805 only contains a single LoRA within their qkv_proj layer.
806
807 During inference with Tensor Parallel, the weights of lora_b
vllm/model_executor/models/chatglm.py
+2 -2
@@ -760,7 +760,7 @@ class ChatGLMForCausalLM(ChatGLMBaseModel, SupportsLoRA, SupportsPP,
760 config = vllm_config.model_config.hf_config
761 # Initialize VL
762 if hasattr(config, "visual"):
763 - return ChatGLM(vllm_config=vllm_config, prefix=prefix)
763 + return ChatGLMV(vllm_config=vllm_config, prefix=prefix)
764 # Initialize LLM
765 else:
766 - return ChatGLMV(vllm_config=vllm_config, prefix=prefix)
766 + return ChatGLM(vllm_config=vllm_config, prefix=prefix)