A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 14:45 UTC
[CI/Build] Fix CI LoRA failure (#16270)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Apr 9, 2025 at 09:13 UTC
86c3369eb888fc57c8edcbbfd0bc6f22a606f79c
8 files changed
+19
-13
tests/lora/conftest.py
+12
@@ -256,3 +256,15 @@ def run_with_both_engines_lora(request, monkeypatch):
256
monkeypatch.setenv('VLLM_USE_V1', '0')
257
258
yield
259
+
260
+
261
+@pytest.fixture
262
+def reset_default_device():
263
+ """
264
+ Some tests, such as `test_punica_ops.py`, explicitly set the
265
+ default device, which can affect subsequent tests. Adding this fixture
266
+ helps avoid this problem.
267
+ """
268
+ original_device = torch.get_default_device()
269
+ yield
270
+ torch.set_default_device(original_device)
tests/lora/test_baichuan.py
-1
@@ -73,7 +73,6 @@ def test_baichuan_tensor_parallel_equality(baichuan_lora_files,
73
max_num_seqs=16,
74
max_loras=4,
75
max_lora_rank=64,
76
- tensor_parallel_size=1,
76
trust_remote_code=True,
77
fully_sharded_loras=fully_sharded)
78
output_tp1 = do_sample(llm_tp1, baichuan_lora_files, lora_id=1)
tests/lora/test_chatglm3_tp.py
-1
@@ -61,7 +61,6 @@ def test_chatglm3_lora(chatglm3_lora_files):
61
enable_lora=True,
62
max_loras=4,
63
max_lora_rank=64,
64
- tensor_parallel_size=1,
64
trust_remote_code=True,
65
enable_chunked_prefill=True)
66
tests/lora/test_layers.py
+1
-1
@@ -65,7 +65,7 @@ VOCAB_PARALLEL_EMBEDDING_TEST_NUM_RANDOM_SEEDS = 128
65
66
67
@pytest.fixture(autouse=True)
68
-def clean_cache():
68
+def clean_cache_reset_device(reset_default_device):
69
# Release any memory we might be holding on to. CI runs OOMs otherwise.
70
from vllm.lora.ops.triton_ops.utils import (_LORA_A_PTR_DICT,
71
_LORA_B_PTR_DICT)
tests/lora/test_llama_tp.py
-1
@@ -88,7 +88,6 @@ def test_llama_lora(sql_lora_files):
88
# also test odd max_num_seqs
89
max_num_seqs=13,
90
max_loras=4,
91
- tensor_parallel_size=1,
91
enable_chunked_prefill=True)
92
generate_and_test(llm, sql_lora_files)
93
tests/lora/test_punica_ops.py
+5
@@ -13,6 +13,11 @@ from vllm.platforms import current_platform
13
from .utils import PunicaTensors, assert_close, generate_data_for_nslices
14
15
16
+@pytest.fixture(autouse=True)
17
+def reset_device(reset_default_device):
18
+ pass
19
+
20
+
21
# Utility shrink and expand operations used as reference implementations.
22
def sgmv_shrink_for_nslices(
23
nslices: int, inputs_tensor: torch.Tensor,
tests/lora/test_quant_model.py
+1
-8
@@ -78,12 +78,7 @@ def do_sample(llm: vllm.LLM,
78
79
80
@pytest.mark.parametrize("model", MODELS)
81
-@pytest.mark.parametrize("tp_size", [1])
82
-def test_quant_model_lora(tinyllama_lora_files, num_gpus_available, model,
83
- tp_size):
84
- if num_gpus_available < tp_size and \
85
- tp_size > 1 and current_platform.is_cuda_alike():
86
- pytest.skip(f"Not enough GPUs for tensor parallelism {tp_size}")
81
+def test_quant_model_lora(tinyllama_lora_files, model):
82
83
llm = vllm.LLM(
84
model=model.model_path,
@@ -91,7 +86,6 @@ def test_quant_model_lora(tinyllama_lora_files, num_gpus_available, model,
86
max_num_seqs=16,
87
max_loras=4,
88
max_model_len=400,
94
- tensor_parallel_size=tp_size,
89
gpu_memory_utilization=0.2, #avoid OOM
90
quantization=model.quantization,
91
trust_remote_code=True,
@@ -185,7 +179,6 @@ def test_quant_model_tp_equality(tinyllama_lora_files, num_gpus_available,
179
enable_lora=True,
180
max_num_seqs=16,
181
max_loras=4,
188
- tensor_parallel_size=1,
182
gpu_memory_utilization=0.2, #avoid OOM
183
quantization=model.quantization,
184
trust_remote_code=True,
tests/lora/test_transfomers_model.py
-1
@@ -53,7 +53,6 @@ def test_ilama_lora(ilama_lora_files):
53
enable_lora=True,
54
max_loras=4,
55
max_lora_rank=16,
56
- tensor_parallel_size=1,
56
trust_remote_code=True,
57
enable_chunked_prefill=True)
58