A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[CI/Build] Remove redundant LoRA model tests (#23706)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Aug 27, 2025 at 13:51 UTC
9de25c294b92e42a12d1fbbb3ab3f633fa80291c
3 files changed
-188
tests/lora/conftest.py
-5
@@ -216,11 +216,6 @@ def tinyllama_lora_files():
216
return snapshot_download(repo_id="jashing/tinyllama-colorist-lora")
217
218
219
-@pytest.fixture(scope="session")
220
-def phi2_lora_files():
221
- return snapshot_download(repo_id="isotr0py/phi-2-test-sql-lora")
222
-
223
-
219
@pytest.fixture
220
def reset_default_device():
221
"""
tests/lora/test_baichuan.py
deleted
-112
@@ -1,112 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-import pytest
5
-
6
-import vllm
7
-from vllm.distributed import cleanup_dist_env_and_memory
8
-from vllm.lora.request import LoRARequest
9
-
10
-MODEL_PATH = "baichuan-inc/Baichuan-7B"
11
-
12
-PROMPT_TEMPLATE = """I want you to act as a SQL terminal in front of an example database, you need only to return the sql command to me.Below is an instruction that describes a task, Write a response that appropriately completes the request.\n"\n##Instruction:\nconcert_singer contains tables such as stadium, singer, concert, singer_in_concert. Table stadium has columns such as Stadium_ID, Location, Name, Capacity, Highest, Lowest, Average. Stadium_ID is the primary key.\nTable singer has columns such as Singer_ID, Name, Country, Song_Name, Song_release_year, Age, Is_male. Singer_ID is the primary key.\nTable concert has columns such as concert_ID, concert_Name, Theme, Stadium_ID, Year. concert_ID is the primary key.\nTable singer_in_concert has columns such as concert_ID, Singer_ID. concert_ID is the primary key.\nThe Stadium_ID of concert is the foreign key of Stadium_ID of stadium.\nThe Singer_ID of singer_in_concert is the foreign key of Singer_ID of singer.\nThe concert_ID of singer_in_concert is the foreign key of concert_ID of concert.\n\n###Input:\n{query}\n\n###Response:""" # noqa: E501
13
-
14
-
15
-def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> list[str]:
16
- prompts = [
17
- PROMPT_TEMPLATE.format(query="How many singers do we have?"),
18
- PROMPT_TEMPLATE.format(
19
- query=
20
- "What is the average, minimum, and maximum age of all singers from France?" # noqa: E501
21
- ),
22
- PROMPT_TEMPLATE.format(
23
- query=
24
- "Show name, country, age for all singers ordered by age from the oldest to the youngest." # noqa: E501
25
- ),
26
- ]
27
- print(prompts)
28
- sampling_params = vllm.SamplingParams(temperature=0, max_tokens=256)
29
- outputs = llm.generate(
30
- prompts,
31
- sampling_params,
32
- lora_request=LoRARequest(str(lora_id), lora_id, lora_path)
33
- if lora_id else None)
34
- # Print the outputs.
35
- generated_texts: list[str] = []
36
- for output in outputs:
37
- prompt = output.prompt
38
- generated_text = output.outputs[0].text.strip()
39
- generated_texts.append(generated_text)
40
- print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
41
- return generated_texts
42
-
43
-
44
-def test_baichuan_lora(baichuan_lora_files):
45
- llm = vllm.LLM(MODEL_PATH,
46
- max_model_len=1024,
47
- enable_lora=True,
48
- max_loras=4,
49
- max_lora_rank=64,
50
- trust_remote_code=True)
51
-
52
- expected_lora_output = [
53
- "SELECT count(*) FROM singer",
54
- "SELECT avg(age) , min(age) , max(age) FROM singer WHERE Country = 'France'", # noqa: E501
55
- "SELECT name , country , age FROM singer ORDER BY age ASC",
56
- ]
57
-
58
- output1 = do_sample(llm, baichuan_lora_files, lora_id=1)
59
- for i in range(len(expected_lora_output)):
60
- assert output1[i] == expected_lora_output[i]
61
- output2 = do_sample(llm, baichuan_lora_files, lora_id=2)
62
- for i in range(len(expected_lora_output)):
63
- assert output2[i] == expected_lora_output[i]
64
-
65
-
66
-@pytest.mark.parametrize("fully_sharded", [True, False])
67
-def test_baichuan_tensor_parallel_equality(baichuan_lora_files,
68
- num_gpus_available, fully_sharded):
69
- if num_gpus_available < 4:
70
- pytest.skip(f"Not enough GPUs for tensor parallelism {4}")
71
-
72
- llm_tp1 = vllm.LLM(MODEL_PATH,
73
- enable_lora=True,
74
- max_num_seqs=16,
75
- max_loras=4,
76
- max_lora_rank=64,
77
- trust_remote_code=True,
78
- fully_sharded_loras=fully_sharded)
79
- output_tp1 = do_sample(llm_tp1, baichuan_lora_files, lora_id=1)
80
-
81
- del llm_tp1
82
- cleanup_dist_env_and_memory()
83
-
84
- llm_tp2 = vllm.LLM(MODEL_PATH,
85
- enable_lora=True,
86
- max_num_seqs=16,
87
- max_loras=4,
88
- max_lora_rank=64,
89
- tensor_parallel_size=2,
90
- trust_remote_code=True,
91
- fully_sharded_loras=fully_sharded)
92
- output_tp2 = do_sample(llm_tp2, baichuan_lora_files, lora_id=2)
93
-
94
- del llm_tp2
95
- cleanup_dist_env_and_memory()
96
-
97
- assert output_tp1 == output_tp2
98
-
99
- llm_tp4 = vllm.LLM(MODEL_PATH,
100
- enable_lora=True,
101
- max_num_seqs=16,
102
- max_loras=4,
103
- max_lora_rank=64,
104
- tensor_parallel_size=4,
105
- trust_remote_code=True,
106
- fully_sharded_loras=fully_sharded)
107
- output_tp4 = do_sample(llm_tp4, baichuan_lora_files, lora_id=2)
108
-
109
- del llm_tp4
110
- cleanup_dist_env_and_memory()
111
-
112
- assert output_tp1 == output_tp4
tests/lora/test_phi.py
deleted
-71
@@ -1,71 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-import vllm
5
-from vllm.lora.request import LoRARequest
6
-
7
-MODEL_PATH = "microsoft/phi-2"
8
-
9
-PROMPT_TEMPLATE = "### Instruct: {sql_prompt}\n\n### Context: {context}\n\n### Output:" # noqa: E501
10
-
11
-
12
-def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> list[str]:
13
- prompts = [
14
- PROMPT_TEMPLATE.format(
15
- sql_prompt=
16
- "Which catalog publisher has published the most catalogs?",
17
- context="CREATE TABLE catalogs (catalog_publisher VARCHAR);"),
18
- PROMPT_TEMPLATE.format(
19
- sql_prompt=
20
- "Which trip started from the station with the largest dock count? Give me the trip id.", # noqa: E501
21
- context=
22
- "CREATE TABLE trip (id VARCHAR, start_station_id VARCHAR); CREATE TABLE station (id VARCHAR, dock_count VARCHAR);" # noqa: E501
23
- ),
24
- PROMPT_TEMPLATE.format(
25
- sql_prompt=
26
- "How many marine species are found in the Southern Ocean?", # noqa: E501
27
- context=
28
- "CREATE TABLE marine_species (name VARCHAR(50), common_name VARCHAR(50), location VARCHAR(50));" # noqa: E501
29
- ),
30
- ]
31
- sampling_params = vllm.SamplingParams(temperature=0,
32
- max_tokens=64,
33
- stop="### End")
34
- outputs = llm.generate(
35
- prompts,
36
- sampling_params,
37
- lora_request=LoRARequest(str(lora_id), lora_id, lora_path)
38
- if lora_id else None,
39
- )
40
- # Print the outputs.
41
- generated_texts: list[str] = []
42
- for output in outputs:
43
- prompt = output.prompt
44
- generated_text = output.outputs[0].text.strip()
45
- generated_texts.append(generated_text)
46
- print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
47
- return generated_texts
48
-
49
-
50
-def test_phi2_lora(phi2_lora_files):
51
- # We enable enforce_eager=True here to reduce VRAM usage for lora-test CI,
52
- # Otherwise, the lora-test will fail due to CUDA OOM.
53
- llm = vllm.LLM(MODEL_PATH,
54
- max_model_len=1024,
55
- enable_lora=True,
56
- max_loras=2,
57
- enforce_eager=True,
58
- enable_chunked_prefill=True)
59
-
60
- expected_lora_output = [
61
- "SELECT catalog_publisher, COUNT(*) as num_catalogs FROM catalogs GROUP BY catalog_publisher ORDER BY num_catalogs DESC LIMIT 1;", # noqa: E501
62
- "SELECT trip.id FROM trip JOIN station ON trip.start_station_id = station.id WHERE station.dock_count = (SELECT MAX(dock_count) FROM station);", # noqa: E501
63
- "SELECT COUNT(*) FROM marine_species WHERE location = 'Southern Ocean';", # noqa: E501
64
- ]
65
-
66
- output1 = do_sample(llm, phi2_lora_files, lora_id=1)
67
- for i in range(len(expected_lora_output)):
68
- assert output1[i].startswith(expected_lora_output[i])
69
- output2 = do_sample(llm, phi2_lora_files, lora_id=2)
70
- for i in range(len(expected_lora_output)):
71
- assert output2[i].startswith(expected_lora_output[i])