A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 06:00 UTC
[CI/Build] Fix LoRA test (#19350)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Jun 9, 2025 at 17:52 UTC
95a6568b5c366359fabf519f7863021377cc0dd6
5 files changed
+11
-63
tests/lora/conftest.py
-5
@@ -164,11 +164,6 @@ def mixtral_lora_files():
164
return snapshot_download(repo_id="SangBinCho/mixtral-lora")
165
166
167
-@pytest.fixture(scope="session")
168
-def gemma_lora_files():
169
- return snapshot_download(repo_id="wskwon/gemma-7b-test-lora")
170
-
171
-
167
@pytest.fixture(scope="session")
168
def chatglm3_lora_files():
169
return snapshot_download(repo_id="jeeejeee/chatglm3-text2sql-spider")
tests/lora/test_llama_tp.py
-34
@@ -4,9 +4,6 @@ import subprocess
4
import sys
5
from typing import Union
6
7
-import pytest
8
-import ray
9
-
7
import vllm
8
from vllm import LLM
9
from vllm.lora.request import LoRARequest
@@ -121,37 +118,6 @@ def test_llama_lora(sql_lora_files):
118
generate_and_test(llm, sql_lora_files)
119
120
124
-# Skipping for v1 as v1 doesn't have a good way to expose the num_gpu_blocks
125
-# used by the engine yet.
126
-@pytest.mark.skip_v1
127
-@create_new_process_for_each_test()
128
-def test_llama_lora_warmup(sql_lora_files):
129
- """Test that the LLM initialization works with a warmup LORA path and
130
- is more conservative"""
131
-
132
- @ray.remote(num_gpus=1)
133
- def get_num_gpu_blocks_lora():
134
- llm = vllm.LLM(MODEL_PATH, enable_lora=True, max_num_seqs=16)
135
- num_gpu_blocks_lora_warmup = llm.llm_engine.cache_config.num_gpu_blocks
136
- return num_gpu_blocks_lora_warmup
137
-
138
- @ray.remote(num_gpus=1)
139
- def get_num_gpu_blocks_no_lora():
140
- llm = vllm.LLM(MODEL_PATH, max_num_seqs=16)
141
- num_gpu_blocks_no_lora_warmup = (
142
- llm.llm_engine.cache_config.num_gpu_blocks)
143
- return num_gpu_blocks_no_lora_warmup
144
-
145
- num_gpu_blocks_lora_warmup = ray.get(get_num_gpu_blocks_lora.remote())
146
- num_gpu_blocks_no_lora_warmup = ray.get(
147
- get_num_gpu_blocks_no_lora.remote())
148
- assert num_gpu_blocks_lora_warmup < num_gpu_blocks_no_lora_warmup, (
149
- "The warmup with lora should be more "
150
- "conservative than without lora, therefore the number of "
151
- "memory blocks for the KV cache should be "
152
- "less when using lora than when not using lora")
153
-
154
-
121
@multi_gpu_test(num_gpus=4)
122
@create_new_process_for_each_test()
123
def test_llama_lora_tp4(sql_lora_files):
tests/lora/test_lora_functions.py
-7
@@ -15,13 +15,6 @@ MODEL_PATH = "meta-llama/Llama-2-7b-hf"
15
LORA_MODULE_PATH = "yard1/llama-2-7b-sql-lora-test"
16
LORA_RANK = 8
17
18
-# @pytest.fixture(autouse=True)
19
-# def v1(run_with_both_engines_lora):
20
-# # Simple autouse wrapper to run both engines for each test
21
-# # This can be promoted up to conftest.py to run for every
22
-# # test in a package
23
-# pass
24
-
18
19
def make_lora_request(lora_id: int):
20
return LoRARequest(lora_name=f"{lora_id}",
tests/lora/test_phi.py
+1
-9
@@ -11,14 +11,6 @@ MODEL_PATH = "microsoft/phi-2"
11
PROMPT_TEMPLATE = "### Instruct: {sql_prompt}\n\n### Context: {context}\n\n### Output:" # noqa: E501
12
13
14
-@pytest.fixture(autouse=True)
15
-def v1(run_with_both_engines_lora):
16
- # Simple autouse wrapper to run both engines for each test
17
- # This can be promoted up to conftest.py to run for every
18
- # test in a package
19
- pass
20
-
21
-
14
def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> list[str]:
15
prompts = [
16
PROMPT_TEMPLATE.format(
@@ -59,7 +51,7 @@ def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> list[str]:
51
52
# Skipping for V1 for now as we are hitting,
53
# "Head size 80 is not supported by FlashAttention." error.
62
-@pytest.mark.skip_v1
54
+@pytest.mark.skip(reason="Head size 80 is not supported by FlashAttention")
55
def test_phi2_lora(phi2_lora_files):
56
# We enable enforce_eager=True here to reduce VRAM usage for lora-test CI,
57
# Otherwise, the lora-test will fail due to CUDA OOM.
tests/lora/test_worker.py
+10
-8
@@ -16,6 +16,8 @@ from vllm.lora.request import LoRARequest
16
from vllm.v1.worker.gpu_worker import Worker as V1Worker
17
from vllm.worker.worker import Worker
18
19
+NUM_LORAS = 16
20
+
21
22
@patch.dict(os.environ, {"RANK": "0"})
23
def test_worker_apply_lora(sql_lora_files):
@@ -58,12 +60,12 @@ def test_worker_apply_lora(sql_lora_files):
60
device_config=DeviceConfig("cuda"),
61
cache_config=CacheConfig(
62
block_size=16,
61
- gpu_memory_utilization=1.0,
63
swap_space=0,
64
cache_dtype="auto",
65
),
65
- lora_config=LoRAConfig(max_lora_rank=8, max_cpu_loras=32,
66
- max_loras=32),
66
+ lora_config=LoRAConfig(max_lora_rank=8,
67
+ max_cpu_loras=NUM_LORAS,
68
+ max_loras=NUM_LORAS),
69
)
70
worker = worker_cls(
71
vllm_config=vllm_config,
@@ -78,9 +80,9 @@ def test_worker_apply_lora(sql_lora_files):
80
set_active_loras(worker, [])
81
assert worker.list_loras() == set()
82
81
- n_loras = 32
83
lora_requests = [
83
- LoRARequest(str(i + 1), i + 1, sql_lora_files) for i in range(n_loras)
84
+ LoRARequest(str(i + 1), i + 1, sql_lora_files)
85
+ for i in range(NUM_LORAS)
86
]
87
88
set_active_loras(worker, lora_requests)
@@ -89,12 +91,12 @@ def test_worker_apply_lora(sql_lora_files):
91
for lora_request in lora_requests
92
}
93
92
- for i in range(32):
94
+ for i in range(NUM_LORAS):
95
random.seed(i)
96
iter_lora_requests = random.choices(lora_requests,
95
- k=random.randint(1, n_loras))
97
+ k=random.randint(1, NUM_LORAS))
98
random.shuffle(iter_lora_requests)
97
- iter_lora_requests = iter_lora_requests[:-random.randint(0, n_loras)]
99
+ iter_lora_requests = iter_lora_requests[:-random.randint(0, NUM_LORAS)]
100
set_active_loras(worker, lora_requests)
101
assert worker.list_loras().issuperset(
102
{lora_request.lora_int_id