A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 16:00 UTC
[CI/Build] Clean up LoRA tests (#15867)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Apr 2, 2025 at 00:28 UTC
dfa82e2a3da027433bde4fe4eb67ce169a30b9dd
5 files changed
+27
-200
tests/lora/test_baichuan.py
-8
@@ -11,14 +11,6 @@ MODEL_PATH = "baichuan-inc/Baichuan-7B"
11
PROMPT_TEMPLATE = """I want you to act as a SQL terminal in front of an example database, you need only to return the sql command to me.Below is an instruction that describes a task, Write a response that appropriately completes the request.\n"\n##Instruction:\nconcert_singer contains tables such as stadium, singer, concert, singer_in_concert. Table stadium has columns such as Stadium_ID, Location, Name, Capacity, Highest, Lowest, Average. Stadium_ID is the primary key.\nTable singer has columns such as Singer_ID, Name, Country, Song_Name, Song_release_year, Age, Is_male. Singer_ID is the primary key.\nTable concert has columns such as concert_ID, concert_Name, Theme, Stadium_ID, Year. concert_ID is the primary key.\nTable singer_in_concert has columns such as concert_ID, Singer_ID. concert_ID is the primary key.\nThe Stadium_ID of concert is the foreign key of Stadium_ID of stadium.\nThe Singer_ID of singer_in_concert is the foreign key of Singer_ID of singer.\nThe concert_ID of singer_in_concert is the foreign key of concert_ID of concert.\n\n###Input:\n{query}\n\n###Response:""" # noqa: E501
12
13
14
-@pytest.fixture(autouse=True)
15
-def v1(run_with_both_engines_lora):
16
- # Simple autouse wrapper to run both engines for each test
17
- # This can be promoted up to conftest.py to run for every
18
- # test in a package
19
- pass
20
-
21
-
14
def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> list[str]:
15
prompts = [
16
PROMPT_TEMPLATE.format(query="How many singers do we have?"),
tests/lora/test_gemma.py
deleted
-65
@@ -1,65 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-
3
-import pytest
4
-
5
-import vllm
6
-from vllm.lora.request import LoRARequest
7
-from vllm.platforms import current_platform
8
-
9
-MODEL_PATH = "google/gemma-7b"
10
-
11
-
12
-@pytest.fixture(autouse=True)
13
-def v1(run_with_both_engines_lora):
14
- # Simple autouse wrapper to run both engines for each test
15
- # This can be promoted up to conftest.py to run for every
16
- # test in a package
17
- pass
18
-
19
-
20
-def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> list[str]:
21
- prompts = [
22
- "Quote: Imagination is",
23
- "Quote: Be yourself;",
24
- "Quote: Painting is poetry that is seen rather than felt,",
25
- ]
26
- sampling_params = vllm.SamplingParams(temperature=0, max_tokens=32)
27
- outputs = llm.generate(
28
- prompts,
29
- sampling_params,
30
- lora_request=LoRARequest(str(lora_id), lora_id, lora_path)
31
- if lora_id else None)
32
- # Print the outputs.
33
- generated_texts: list[str] = []
34
- for output in outputs:
35
- prompt = output.prompt
36
- generated_text = output.outputs[0].text.strip()
37
- generated_texts.append(generated_text)
38
- print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
39
- return generated_texts
40
-
41
-
42
-# The V1 lora test for this model requires more than 24GB.
43
-@pytest.mark.skip_v1
44
-@pytest.mark.xfail(current_platform.is_rocm(),
45
- reason="There can be output mismatch on ROCm")
46
-def test_gemma_lora(gemma_lora_files):
47
- llm = vllm.LLM(MODEL_PATH,
48
- max_model_len=1024,
49
- enable_lora=True,
50
- max_loras=4,
51
- enable_chunked_prefill=True)
52
-
53
- expected_lora_output = [
54
- "more important than knowledge.\nAuthor: Albert Einstein\n",
55
- "everyone else is already taken.\nAuthor: Oscar Wilde\n",
56
- "and poetry is painting that is felt rather than seen.\n"
57
- "Author: Leonardo da Vinci\n",
58
- ]
59
-
60
- output1 = do_sample(llm, gemma_lora_files, lora_id=1)
61
- for i in range(len(expected_lora_output)):
62
- assert output1[i].startswith(expected_lora_output[i])
63
- output2 = do_sample(llm, gemma_lora_files, lora_id=2)
64
- for i in range(len(expected_lora_output)):
65
- assert output2[i].startswith(expected_lora_output[i])
tests/lora/test_layers.py
+23
-117
@@ -19,7 +19,6 @@ from vllm.lora.fully_sharded_layers import (
19
# yapf conflicts with isort for this block
20
# yapf: disable
21
from vllm.lora.layers import (BaseLayerWithLoRA, ColumnParallelLinearWithLoRA,
22
- LinearScalingRotaryEmbeddingWithLoRA,
22
LogitsProcessorWithLoRA, LoRAMapping,
23
MergedColumnParallelLinearWithLoRA,
24
MergedQKVParallelLinearWithLoRA,
@@ -28,8 +27,7 @@ from vllm.lora.layers import (BaseLayerWithLoRA, ColumnParallelLinearWithLoRA,
27
RowParallelLinearWithLoRA,
28
VocabParallelEmbeddingWithLoRA)
29
# yapf: enable
31
-from vllm.lora.models import (LongContextLoRAContext, LoRALayerWeights,
32
- PackedLoRALayerWeights)
30
+from vllm.lora.models import LoRALayerWeights, PackedLoRALayerWeights
31
from vllm.lora.punica_wrapper import get_punica_wrapper
32
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
33
MergedColumnParallelLinear,
@@ -37,7 +35,6 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
35
ReplicatedLinear,
36
RowParallelLinear)
37
from vllm.model_executor.layers.logits_processor import LogitsProcessor
40
-from vllm.model_executor.layers.rotary_embedding import get_rope
38
from vllm.model_executor.layers.vocab_parallel_embedding import (
39
ParallelLMHead, VocabParallelEmbedding, get_masked_input_and_mask)
40
from vllm.model_executor.utils import set_random_seed
@@ -59,28 +56,16 @@ DEVICES = ([
56
f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
57
] if current_platform.is_cuda_alike() else ["cpu"])
58
62
-#For GPU, we will launch different triton kernels between the prefill and decode
63
-# stages, so we need to verify this. prefill stage(True) or decode stage(False)
59
+# prefill stage(True) or decode stage(False)
60
STAGES = [True, False]
61
66
-# With the inclusion of V1 tests (look at the run_with_both_engines_lora),
67
-# the tests in this file run twice, once with the V0 engine and then with
68
-# the V1 engine.
69
-# The NUM_RANDOM_SEEDS value was set to 10 before. It is cut to half
70
-# with the inclusion of V1 tests to maintain the CI test times.
71
-NUM_RANDOM_SEEDS = 5
72
-# The VOCAB_PARALLEL_EMBEDDING_TEST_NUM_RANDOM_SEEDS value was set to
73
-# 256 before. It is cut to half with the inclusion of V1 tests to maintain
74
-# the CI test times.
62
+NUM_RANDOM_SEEDS = 10
63
+
64
VOCAB_PARALLEL_EMBEDDING_TEST_NUM_RANDOM_SEEDS = 128
65
66
67
@pytest.fixture(autouse=True)
79
-def v1(run_with_both_engines_lora):
80
- # Simple autouse wrapper to run both engines for each test
81
- # This can be promoted up to conftest.py to run for every
82
- # test in a package
83
-
68
+def clean_cache():
69
# Release any memory we might be holding on to. CI runs OOMs otherwise.
70
from vllm.lora.ops.triton_ops.utils import (_LORA_A_PTR_DICT,
71
_LORA_B_PTR_DICT)
@@ -90,6 +75,24 @@ def v1(run_with_both_engines_lora):
75
yield
76
77
78
+@pytest.fixture(autouse=True)
79
+def skip_cuda_with_stage_false(request):
80
+ """
81
+ On cuda-like platforms, we use the same kernels for prefill and decode
82
+ stage, and 'stage' is generally ignored, so we only need to test once.
83
+ """
84
+ if current_platform.is_cuda_alike():
85
+ try:
86
+ if hasattr(request.node, "callspec") and hasattr(
87
+ request.node.callspec, "params"):
88
+ params = request.node.callspec.params
89
+ if "stage" in params and params["stage"] is False:
90
+ pytest.skip("Skip test when stage=False")
91
+ except Exception:
92
+ pass
93
+ yield
94
+
95
+
96
def get_random_id_to_index(num_loras: int,
97
num_slots: int,
98
log: bool = True) -> list[Optional[int]]:
@@ -1011,103 +1014,6 @@ def test_column_parallel_packed(dist_init, num_loras, repeats, fully_shard,
1014
atol=atol)
1015
1016
1014
-@torch.inference_mode()
1015
-@pytest.mark.parametrize("num_loras", [1, 8])
1016
-@pytest.mark.parametrize("device", ["cuda"])
1017
-@pytest.mark.parametrize("scaling_factors", [(1.0, ), (4.0, ), (4.0, 8.0),
1018
- (6.0, 1.0)])
1019
-@pytest.mark.parametrize("max_position", [11, 4096, 32768])
1020
-@pytest.mark.parametrize("is_neox_style", [True, False])
1021
-@pytest.mark.parametrize("rotary_dim", [None, 32])
1022
-@pytest.mark.parametrize("head_size", [32, 108])
1023
-@pytest.mark.parametrize("seq_len", [11, 1024])
1024
-@pytest.mark.skipif(not current_platform.is_cuda_alike(),
1025
- reason="Only CUDA backends are supported")
1026
-def test_rotary_embedding_long_context(dist_init, num_loras, device,
1027
- scaling_factors, max_position,
1028
- is_neox_style, rotary_dim, head_size,
1029
- seq_len) -> None:
1030
- dtype = torch.float16
1031
- max_loras = 8
1032
- seed = 0
1033
- current_platform.seed_everything(seed)
1034
- torch.set_default_device(device)
1035
- punica_wrapper = get_punica_wrapper(8192, 256, device, max_loras=max_loras)
1036
- assert check_punica_wrapper(punica_wrapper)
1037
- lora_config = LoRAConfig(max_loras=max_loras,
1038
- max_lora_rank=8,
1039
- long_lora_scaling_factors=scaling_factors,
1040
- lora_dtype=dtype)
1041
-
1042
- if rotary_dim is None:
1043
- rotary_dim = head_size
1044
- base = 10000
1045
- batch_size = 5 * num_loras
1046
- num_heads = 7
1047
-
1048
- # Verify lora is equivalent to linear scaling rotary embedding.
1049
- rope = get_rope(
1050
- head_size,
1051
- rotary_dim,
1052
- max_position,
1053
- base,
1054
- is_neox_style,
1055
- )
1056
- lora_rope = LinearScalingRotaryEmbeddingWithLoRA(rope)
1057
- lora_rope.set_mapping(punica_wrapper)
1058
- lora_rope.create_lora_weights(max_loras, lora_config)
1059
- linear_rope = get_rope(head_size, rotary_dim, max_position, base,
1060
- is_neox_style, {
1061
- "rope_type": "linear",
1062
- "factor": scaling_factors
1063
- })
1064
- linear_rope = linear_rope.to(dtype=dtype)
1065
- id_to_index = get_random_id_to_index(num_loras, max_loras)
1066
- _, index_mapping, prompt_mapping = create_random_inputs(
1067
- active_lora_ids=[0],
1068
- num_inputs=batch_size,
1069
- input_size=(1, max_position),
1070
- input_range=(0, lora_config.lora_extra_vocab_size),
1071
- input_type=torch.float16,
1072
- device=device)
1073
-
1074
- lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
1075
- long_lora_context = LongContextLoRAContext(list(scaling_factors),
1076
- rotary_dim)
1077
-
1078
- next_expected_offset = 0
1079
- # Make sure the offset is correct.
1080
- scaling_factor_to_offset = lora_rope.scaling_factor_to_offset
1081
- for scaling_factor, offset in scaling_factor_to_offset.items():
1082
- assert offset == next_expected_offset
1083
- next_expected_offset += scaling_factor * max_position
1084
-
1085
- for i in range(len(scaling_factors)):
1086
- long_lora_context.offsets_by_lora_id[i] = scaling_factor_to_offset.get(
1087
- scaling_factors[i], 0)
1088
- punica_wrapper.update_metadata(
1089
- lora_mapping,
1090
- id_to_index,
1091
- max_loras,
1092
- 512,
1093
- lora_config.lora_extra_vocab_size,
1094
- long_lora_context=long_lora_context,
1095
- )
1096
- # lora_rope.set_mapping(*mapping_info)
1097
-
1098
- positions = torch.randint(0, max_position, (batch_size, seq_len))
1099
- query = torch.randn(batch_size,
1100
- seq_len,
1101
- num_heads * head_size,
1102
- dtype=dtype)
1103
- key = torch.randn_like(query)
1104
- ref_q, ref_k = linear_rope(positions, query, key)
1105
- actual_q, actual_k = lora_rope(positions, query, key)
1106
-
1107
- torch.allclose(ref_q, actual_q)
1108
- torch.allclose(ref_k, actual_k)
1109
-
1110
-
1017
@pytest.mark.parametrize("tp_size", [1, 2, 4, 8])
1018
@pytest.mark.parametrize(
1019
"seed", list(range(VOCAB_PARALLEL_EMBEDDING_TEST_NUM_RANDOM_SEEDS)))
tests/lora/test_minicpmv_tp.py
+4
@@ -78,6 +78,8 @@ def test_minicpmv_lora(minicpmv_lora_files):
78
assert EXPECTED_OUTPUT[i].startswith(output2[i])
79
80
81
+@pytest.mark.skipif(current_platform.is_cuda_alike(),
82
+ reason="Skipping to avoid redundant model tests")
83
@pytest.mark.xfail(
84
current_platform.is_rocm(),
85
reason="MiniCPM-V dependency xformers incompatible with ROCm")
@@ -99,6 +101,8 @@ def test_minicpmv_tp4_wo_fully_sharded_loras(minicpmv_lora_files):
101
assert EXPECTED_OUTPUT[i].startswith(output_tp[i])
102
103
104
+@pytest.mark.skipif(current_platform.is_cuda_alike(),
105
+ reason="Skipping to avoid redundant model tests")
106
@pytest.mark.xfail(
107
current_platform.is_rocm(),
108
reason="MiniCPM-V dependency xformers incompatible with ROCm")
tests/lora/test_transfomers_model.py
-10
@@ -1,7 +1,5 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
-import pytest
4
-
3
import vllm
4
from vllm.lora.request import LoRARequest
5
@@ -18,14 +16,6 @@ EXPECTED_LORA_OUTPUT = [
16
]
17
18
21
-@pytest.fixture(autouse=True)
22
-def v1(run_with_both_engines_lora):
23
- # Simple autouse wrapper to run both engines for each test
24
- # This can be promoted up to conftest.py to run for every
25
- # test in a package
26
- pass
27
-
28
-
19
def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> list[str]:
20
prompts = [
21
PROMPT_TEMPLATE.format(query="How many singers do we have?"),