@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 39e63dec7c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 12, 08:30 UTC

[LoRA] Cleanup LoRA unused code (#29611)

Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>

Jee Jee Li committed Nov 29, 2025 at 14:52 UTC 39e63dec7c751a28d5a577caf16439e08a878c92
46 files changed +126 -173
examples/offline_inference/multilora_inference.py
+2 -4
@@ -46,7 +46,6 @@ def create_test_prompts(
46 logprobs=1,
47 prompt_logprobs=1,
48 max_tokens=128,
49 - stop_token_ids=[32003],
49 ),
50 LoRARequest("sql-lora", 1, lora_path),
51 ),
@@ -57,7 +56,6 @@ def create_test_prompts(
56 logprobs=1,
57 prompt_logprobs=1,
58 max_tokens=128,
60 - stop_token_ids=[32003],
59 ),
60 LoRARequest("sql-lora2", 2, lora_path),
61 ),
@@ -98,7 +96,7 @@ def initialize_engine() -> LLMEngine:
96 # use the same rank, it is recommended to set this as low as possible.
97 # max_cpu_loras: controls the size of the CPU LoRA cache.
98 engine_args = EngineArgs(
101 - model="meta-llama/Llama-2-7b-hf",
99 + model="meta-llama/Llama-3.2-3B-Instruct",
100 enable_lora=True,
101 max_loras=1,
102 max_lora_rank=8,
@@ -111,7 +109,7 @@ def initialize_engine() -> LLMEngine:
109 def main():
110 """Main function that sets up and runs the prompt processing."""
111 engine = initialize_engine()
114 - lora_path = snapshot_download(repo_id="yard1/llama-2-7b-sql-lora-test")
112 + lora_path = snapshot_download(repo_id="jeeejeee/llama32-3b-text2sql-spider")
113 test_prompts = create_test_prompts(lora_path)
114 process_requests(engine, test_prompts)
115
tests/entrypoints/conftest.py
+3 -3
@@ -188,11 +188,11 @@ number: "1" | "2"
188
189
190 @pytest.fixture(scope="session")
191 -def zephyr_lora_files():
192 - """Download zephyr LoRA files once per test session."""
191 +def qwen3_lora_files():
192 + """Download Qwen3 LoRA files once per test session."""
193 from huggingface_hub import snapshot_download
194
195 - return snapshot_download(repo_id="typeof/zephyr-7b-beta-lora")
195 + return snapshot_download(repo_id="charent/self_cognition_Alice")
196
197
198 @pytest.fixture(scope="session")
tests/entrypoints/openai/test_basic.py
+1 -1
@@ -16,7 +16,7 @@ from vllm.version import __version__ as VLLM_VERSION
16
17 from ...utils import RemoteOpenAIServer
18
19 -MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
19 +MODEL_NAME = "Qwen/Qwen3-0.6B"
20
21
22 @pytest.fixture(scope="module")
tests/entrypoints/openai/test_chat.py
+8
@@ -19,6 +19,14 @@ from ...utils import RemoteOpenAIServer
19 MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
20
21
22 +@pytest.fixture(scope="module")
23 +def zephyr_lora_files():
24 + """Download zephyr LoRA files once per test session."""
25 + from huggingface_hub import snapshot_download
26 +
27 + return snapshot_download(repo_id="typeof/zephyr-7b-beta-lora")
28 +
29 +
30 @pytest.fixture(scope="module")
31 def server(zephyr_lora_files): # noqa: F811
32 args = [
tests/entrypoints/openai/test_chunked_prompt.py
+1 -2
@@ -8,7 +8,7 @@ import pytest_asyncio
8 from ...utils import RemoteOpenAIServer
9
10 # any model with a chat template should work here
11 -MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
11 +MODEL_NAME = "Qwen/Qwen3-0.6B"
12
13
14 @pytest.fixture(scope="module")
@@ -20,7 +20,6 @@ def server():
20 "--max-model-len",
21 "8192",
22 "--enforce-eager",
23 - # lora config below
23 "--max-num-seqs",
24 "128",
25 "--enable-chunked-prefill",
tests/entrypoints/openai/test_lora_adapters.py
+21 -22
@@ -13,9 +13,8 @@ import pytest_asyncio
13 from ...utils import RemoteOpenAIServer
14
15 # any model with a chat template should work here
16 -MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
17 -# technically this needs Mistral-7B-v0.1 as base, but we're not testing
18 -# generation quality here
16 +MODEL_NAME = "Qwen/Qwen3-0.6B"
17 +
18
19 BADREQUEST_CASES = [
20 (
@@ -33,11 +32,11 @@ BADREQUEST_CASES = [
32
33
34 @pytest.fixture(scope="module", params=[True])
36 -def server_with_lora_modules_json(request, zephyr_lora_files):
35 +def server_with_lora_modules_json(request, qwen3_lora_files):
36 # Define the json format LoRA module configurations
37 lora_module_1 = {
39 - "name": "zephyr-lora",
40 - "path": zephyr_lora_files,
38 + "name": "qwen3-lora",
39 + "path": qwen3_lora_files,
40 "base_model_name": MODEL_NAME,
41 }
42
@@ -74,7 +73,7 @@ async def client(server_with_lora_modules_json):
73
74
75 @pytest.mark.asyncio
77 -async def test_static_lora_lineage(client: openai.AsyncOpenAI, zephyr_lora_files):
76 +async def test_static_lora_lineage(client: openai.AsyncOpenAI, qwen3_lora_files):
77 models = await client.models.list()
78 models = models.data
79 served_model = models[0]
@@ -82,17 +81,17 @@ async def test_static_lora_lineage(client: openai.AsyncOpenAI, zephyr_lora_files
81 assert served_model.id == MODEL_NAME
82 assert served_model.root == MODEL_NAME
83 assert served_model.parent is None
85 - assert all(lora_model.root == zephyr_lora_files for lora_model in lora_models)
84 + assert all(lora_model.root == qwen3_lora_files for lora_model in lora_models)
85 assert all(lora_model.parent == MODEL_NAME for lora_model in lora_models)
87 - assert lora_models[0].id == "zephyr-lora"
86 + assert lora_models[0].id == "qwen3-lora"
87
88
89 @pytest.mark.asyncio
91 -async def test_dynamic_lora_lineage(client: openai.AsyncOpenAI, zephyr_lora_files):
90 +async def test_dynamic_lora_lineage(client: openai.AsyncOpenAI, qwen3_lora_files):
91 response = await client.post(
92 "load_lora_adapter",
93 cast_to=str,
95 - body={"lora_name": "zephyr-lora-3", "lora_path": zephyr_lora_files},
94 + body={"lora_name": "qwen3-lora-3", "lora_path": qwen3_lora_files},
95 )
96 # Ensure adapter loads before querying /models
97 assert "success" in response
@@ -100,9 +99,9 @@ async def test_dynamic_lora_lineage(client: openai.AsyncOpenAI, zephyr_lora_file
99 models = await client.models.list()
100 models = models.data
101 dynamic_lora_model = models[-1]
103 - assert dynamic_lora_model.root == zephyr_lora_files
102 + assert dynamic_lora_model.root == qwen3_lora_files
103 assert dynamic_lora_model.parent == MODEL_NAME
105 - assert dynamic_lora_model.id == "zephyr-lora-3"
104 + assert dynamic_lora_model.id == "qwen3-lora-3"
105
106
107 @pytest.mark.asyncio
@@ -134,7 +133,7 @@ async def test_dynamic_lora_invalid_files(client: openai.AsyncOpenAI, tmp_path):
133 async def test_dynamic_lora_badrequests(
134 client: openai.AsyncOpenAI,
135 tmp_path,
137 - zephyr_lora_files,
136 + qwen3_lora_files,
137 test_name: str,
138 config_change: dict,
139 expected_error: str,
@@ -143,7 +142,7 @@ async def test_dynamic_lora_badrequests(
142 test_dir = tmp_path / test_name
143
144 # Copy adapter files
146 - shutil.copytree(zephyr_lora_files, test_dir)
145 + shutil.copytree(qwen3_lora_files, test_dir)
146
147 # Load and modify configuration
148 config_path = test_dir / "adapter_config.json"
@@ -167,7 +166,7 @@ async def test_dynamic_lora_badrequests(
166
167 @pytest.mark.asyncio
168 async def test_multiple_lora_adapters(
170 - client: openai.AsyncOpenAI, tmp_path, zephyr_lora_files
169 + client: openai.AsyncOpenAI, tmp_path, qwen3_lora_files
170 ):
171 """Validate that many loras can be dynamically registered and inferenced
172 with concurrently"""
@@ -178,7 +177,7 @@ async def test_multiple_lora_adapters(
177 await client.post(
178 "load_lora_adapter",
179 cast_to=str,
181 - body={"lora_name": adapter_name, "lora_path": str(zephyr_lora_files)},
180 + body={"lora_name": adapter_name, "lora_path": str(qwen3_lora_files)},
181 )
182 for _ in range(3):
183 await client.completions.create(
@@ -199,7 +198,7 @@ async def test_multiple_lora_adapters(
198
199 @pytest.mark.asyncio
200 async def test_loading_invalid_adapters_does_not_break_others(
202 - client: openai.AsyncOpenAI, tmp_path, zephyr_lora_files
201 + client: openai.AsyncOpenAI, tmp_path, qwen3_lora_files
202 ):
203 invalid_files = tmp_path / "invalid_files"
204 invalid_files.mkdir()
@@ -215,7 +214,7 @@ async def test_loading_invalid_adapters_does_not_break_others(
214 while not stop_good_requests_event.is_set():
215 try:
216 batch = await client.completions.create(
218 - model="zephyr-lora",
217 + model="qwen3-lora",
218 prompt=["Hello there", "Foo bar bazz buzz"],
219 max_tokens=5,
220 )
@@ -254,7 +253,7 @@ async def test_loading_invalid_adapters_does_not_break_others(
253 await client.post(
254 "load_lora_adapter",
255 cast_to=str,
257 - body={"lora_name": "valid", "lora_path": zephyr_lora_files},
256 + body={"lora_name": "valid", "lora_path": qwen3_lora_files},
257 )
258 await client.completions.create(
259 model="valid",
@@ -267,7 +266,7 @@ async def test_loading_invalid_adapters_does_not_break_others(
266 async def test_beam_search_with_lora_adapters(
267 client: openai.AsyncOpenAI,
268 tmp_path,
270 - zephyr_lora_files,
269 + qwen3_lora_files,
270 ):
271 """Validate that async beam search can be used with lora."""
272
@@ -275,7 +274,7 @@ async def test_beam_search_with_lora_adapters(
274 await client.post(
275 "load_lora_adapter",
276 cast_to=str,
278 - body={"lora_name": adapter_name, "lora_path": str(zephyr_lora_files)},
277 + body={"lora_name": adapter_name, "lora_path": str(qwen3_lora_files)},
278 )
279 for _ in range(3):
280 await client.completions.create(
tests/entrypoints/openai/test_models.py
+6 -6
@@ -8,13 +8,13 @@ import pytest_asyncio
8 from ...utils import RemoteOpenAIServer
9
10 # any model with a chat template should work here
11 -MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
11 +MODEL_NAME = "Qwen/Qwen3-0.6B"
12 # technically this needs Mistral-7B-v0.1 as base, but we're not testing
13 # generation quality here
14
15
16 @pytest.fixture(scope="module")
17 -def server(zephyr_lora_files):
17 +def server(qwen3_lora_files):
18 args = [
19 # use half precision for speed and memory savings in CI environment
20 "--dtype",
@@ -25,7 +25,7 @@ def server(zephyr_lora_files):
25 # lora config below
26 "--enable-lora",
27 "--lora-modules",
28 - f"zephyr-lora={zephyr_lora_files}",
28 + f"qwen3-lora={qwen3_lora_files}",
29 "--max-lora-rank",
30 "64",
31 "--max-cpu-loras",
@@ -45,12 +45,12 @@ async def client(server):
45
46
47 @pytest.mark.asyncio
48 -async def test_check_models(client: openai.AsyncOpenAI, zephyr_lora_files):
48 +async def test_check_models(client: openai.AsyncOpenAI, qwen3_lora_files):
49 models = await client.models.list()
50 models = models.data
51 served_model = models[0]
52 lora_models = models[1:]
53 assert served_model.id == MODEL_NAME
54 assert served_model.root == MODEL_NAME
55 - assert all(lora_model.root == zephyr_lora_files for lora_model in lora_models)
56 - assert lora_models[0].id == "zephyr-lora"
55 + assert all(lora_model.root == qwen3_lora_files for lora_model in lora_models)
56 + assert lora_models[0].id == "qwen3-lora"
tests/entrypoints/openai/test_orca_metrics.py
+3 -2
@@ -8,7 +8,7 @@ import pytest_asyncio
8 from ...utils import RemoteOpenAIServer
9
10 # any model with a chat template should work here
11 -MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
11 +MODEL_NAME = "Qwen/Qwen3-0.6B"
12
13
14 @pytest.fixture(scope="module")
@@ -110,8 +110,9 @@ async def test_single_completion(client: openai.AsyncOpenAI):
110 choice = completion.choices[0]
111 assert len(choice.text) >= 5
112 assert choice.finish_reason == "length"
113 + # When using Qwen3-0.6B, prompt tokens=[9707, 11, 847, 829, 374]
114 assert completion.usage == openai.types.CompletionUsage(
114 - completion_tokens=5, prompt_tokens=6, total_tokens=11
115 + completion_tokens=5, prompt_tokens=5, total_tokens=10
116 )
117
118 # test using token IDs
tests/entrypoints/openai/test_return_tokens_as_ids.py
+3 -3
@@ -11,11 +11,11 @@ from vllm.transformers_utils.tokenizer import get_tokenizer
11
12 from ...utils import RemoteOpenAIServer
13
14 -MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
14 +MODEL_NAME = "Qwen/Qwen3-0.6B"
15
16
17 @pytest.fixture(scope="module")
18 -def default_server_args(zephyr_lora_files):
18 +def default_server_args(qwen3_lora_files):
19 return [
20 # use half precision for speed and memory savings in CI environment
21 "--dtype",
@@ -28,7 +28,7 @@ def default_server_args(zephyr_lora_files):
28 # lora config
29 "--enable-lora",
30 "--lora-modules",
31 - f"zephyr-lora={zephyr_lora_files}",
31 + f"qwen3-lora={qwen3_lora_files}",
32 "--max-lora-rank",
33 "64",
34 "--max-cpu-loras",
tests/entrypoints/openai/test_tokenization.py
+1 -1
@@ -10,7 +10,7 @@ from vllm.transformers_utils.tokenizer import get_tokenizer
10 from ...utils import RemoteOpenAIServer
11
12 # any model with a chat template should work here
13 -MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
13 +MODEL_NAME = "Qwen/Qwen3-0.6B"
14
15
16 @pytest.fixture(scope="module")
tests/entrypoints/openai/test_uds.py
+1 -1
@@ -10,7 +10,7 @@ from vllm.version import __version__ as VLLM_VERSION
10
11 from ...utils import RemoteOpenAIServer
12
13 -MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
13 +MODEL_NAME = "Qwen/Qwen3-0.6B"
14
15
16 @pytest.fixture(scope="module")
tests/entrypoints/sagemaker/conftest.py
-1
@@ -9,7 +9,6 @@ import pytest_asyncio
9 from ...utils import RemoteOpenAIServer
10
11 # Model name constants used across tests
12 -MODEL_NAME_ZEPHYR = "HuggingFaceH4/zephyr-7b-beta"
12 MODEL_NAME_SMOLLM = "HuggingFaceTB/SmolLM2-135M-Instruct"
13 LORA_ADAPTER_NAME_SMOLLM = "jekunz/smollm-135m-lora-fineweb-faroese"
14
tests/lora/conftest.py
+8 -19
@@ -154,23 +154,6 @@ def dummy_model_gate_up() -> nn.Module:
154 return model
155
156
157 -@pytest.fixture(scope="session")
158 -def llama_2_7b_base_huggingface_id():
159 - # used as a base model for testing with sql lora adapter
160 - return "meta-llama/Llama-2-7b-hf"
161 -
162 -
163 -@pytest.fixture(scope="session")
164 -def sql_lora_huggingface_id():
165 - # huggingface repo id is used to test lora runtime downloading.
166 - return "yard1/llama-2-7b-sql-lora-test"
167 -
168 -
169 -@pytest.fixture(scope="session")
170 -def sql_lora_files(sql_lora_huggingface_id):
171 - return snapshot_download(repo_id=sql_lora_huggingface_id)
172 -
173 -
157 @pytest.fixture(scope="session")
158 def mixtral_lora_files():
159 # Note: this module has incorrect adapter_config.json to test
@@ -256,8 +239,14 @@ def qwen3_lora_files():
239
240
241 @pytest.fixture(scope="session")
259 -def llama32_lora_files():
260 - return snapshot_download(repo_id="jeeejeee/llama32-3b-text2sql-spider")
242 +def llama32_lora_huggingface_id():
243 + # huggingface repo id is used to test lora runtime downloading.
244 + return "jeeejeee/llama32-3b-text2sql-spider"
245 +
246 +
247 +@pytest.fixture(scope="session")
248 +def llama32_lora_files(llama32_lora_huggingface_id):
249 + return snapshot_download(repo_id=llama32_lora_huggingface_id)
250
251
252 @pytest.fixture
tests/lora/test_lora_checkpoints.py
+7 -14
@@ -26,8 +26,7 @@ def test_load_checkpoints(
26 chatglm3_lora_files,
27 ):
28 packed_modules_mapping = BaiChuanBaseForCausalLM.packed_modules_mapping
29 - embedding_modules = BaiChuanBaseForCausalLM.embedding_modules
30 - embed_padding_modules = BaiChuanBaseForCausalLM.embedding_padding_modules
29 +
30 expected_lora_lst: list[str] = []
31 for module in BAICHUAN_LORA_MODULES:
32 if module in packed_modules_mapping:
@@ -47,8 +46,7 @@ def test_load_checkpoints(
46 peft_helper=peft_helper,
47 lora_model_id=1,
48 device="cpu",
50 - embedding_modules=embedding_modules,
51 - embedding_padding_modules=embed_padding_modules,
49 + model_vocab_size=64000,
50 )
51 elif lora_name == "baichuan7B-zero":
52 # Test that the target_modules contain prefix
@@ -63,8 +61,7 @@ def test_load_checkpoints(
61 peft_helper=peft_helper,
62 lora_model_id=1,
63 device="cpu",
66 - embedding_modules=embedding_modules,
67 - embedding_padding_modules=embed_padding_modules,
64 + model_vocab_size=64000,
65 )
66 elif lora_name == "baichuan7B-zero-regex":
67 # Test that the `target_modules` in the form of regular expressions,
@@ -78,8 +75,7 @@ def test_load_checkpoints(
75 peft_helper=peft_helper,
76 lora_model_id=1,
77 device="cpu",
81 - embedding_modules=embedding_modules,
82 - embedding_padding_modules=embed_padding_modules,
78 + model_vocab_size=64000,
79 )
80 else:
81 # For the baichuan7B model, load chatglm3-6b's LoRA,
@@ -95,15 +91,13 @@ def test_load_checkpoints(
91 peft_helper=peft_helper,
92 lora_model_id=1,
93 device="cpu",
98 - embedding_modules=embedding_modules,
99 - embedding_padding_modules=embed_padding_modules,
94 + model_vocab_size=64000,
95 )
96
97
98 def test_lora_weights_mapping(baichuan_lora_files):
99 packed_modules_mapping = BaiChuanBaseForCausalLM.packed_modules_mapping
105 - embedding_modules = BaiChuanBaseForCausalLM.embedding_modules
106 - embed_padding_modules = BaiChuanBaseForCausalLM.embedding_padding_modules
100 +
101 expected_lora_lst: list[str] = []
102 for module in BAICHUAN_LORA_MODULES:
103 if module in packed_modules_mapping:
@@ -128,8 +122,7 @@ def test_lora_weights_mapping(baichuan_lora_files):
122 peft_helper=peft_helper,
123 lora_model_id=1,
124 device="cpu",
131 - embedding_modules=embedding_modules,
132 - embedding_padding_modules=embed_padding_modules,
125 + model_vocab_size=64000,
126 weights_mapper=hf_to_vllm_mapper,
127 )
128 for name in lora_model.loras:
tests/lora/test_lora_huggingface.py
+4 -7
@@ -6,10 +6,10 @@ import pytest
6 from vllm.lora.models import LoRAModel
7 from vllm.lora.peft_helper import PEFTHelper
8 from vllm.lora.utils import get_adapter_absolute_path
9 -from vllm.model_executor.models.llama import LlamaForCausalLM
9 +from vllm.model_executor.models.qwen3 import Qwen3ForCausalLM
10
11 # Provide absolute path and huggingface lora ids
12 -lora_fixture_name = ["sql_lora_files", "sql_lora_huggingface_id"]
12 +lora_fixture_name = ["llama32_lora_files", "llama32_lora_huggingface_id"]
13 LLAMA_LORA_MODULES = [
14 "qkv_proj",
15 "o_proj",
@@ -23,9 +23,8 @@ LLAMA_LORA_MODULES = [
23 @pytest.mark.parametrize("lora_fixture_name", lora_fixture_name)
24 def test_load_checkpoints_from_huggingface(lora_fixture_name, request):
25 lora_name = request.getfixturevalue(lora_fixture_name)
26 - packed_modules_mapping = LlamaForCausalLM.packed_modules_mapping
27 - embedding_modules = LlamaForCausalLM.embedding_modules
28 - embed_padding_modules = LlamaForCausalLM.embedding_padding_modules
26 + packed_modules_mapping = Qwen3ForCausalLM.packed_modules_mapping
27 +
28 expected_lora_lst: list[str] = []
29 for module in LLAMA_LORA_MODULES:
30 if module in packed_modules_mapping:
@@ -43,8 +42,6 @@ def test_load_checkpoints_from_huggingface(lora_fixture_name, request):
42 peft_helper=peft_helper,
43 lora_model_id=1,
44 device="cpu",
46 - embedding_modules=embedding_modules,
47 - embedding_padding_modules=embed_padding_modules,
45 )
46
47 # Assertions to ensure the model is loaded correctly
tests/lora/test_lora_manager.py
+6 -11
@@ -34,7 +34,6 @@ EMBEDDING_MODULES = {
34 "lm_head": "output_embeddings",
35 }
36
37 -EMBEDDING_PADDING_MODULES = ["lm_head"]
37
38 DEVICES = (
39 [f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)]
@@ -46,24 +45,22 @@ DEFAULT_DTYPE = torch.get_default_dtype()
45
46
47 @pytest.mark.parametrize("device", DEVICES)
49 -def test_from_lora_tensors(sql_lora_files, device):
50 - tensors = load_file(os.path.join(sql_lora_files, "adapter_model.safetensors"))
48 +def test_from_lora_tensors(qwen3_lora_files, device):
49 + tensors = load_file(os.path.join(qwen3_lora_files, "adapter_model.safetensors"))
50
51 peft_helper = PEFTHelper.from_local_dir(
53 - sql_lora_files, max_position_embeddings=4096
52 + qwen3_lora_files, max_position_embeddings=4096
53 )
54 lora_model = LoRAModel.from_lora_tensors(
55 1,
56 tensors,
57 peft_helper=peft_helper,
58 device=device,
60 - embedding_modules=EMBEDDING_MODULES,
61 - embedding_padding_modules=EMBEDDING_PADDING_MODULES,
59 )
60 for module_name, lora in lora_model.loras.items():
61 assert lora.module_name == module_name
62 assert lora.rank == 8
66 - assert lora.lora_alpha == 16
63 + assert lora.lora_alpha == 32
64 assert lora.lora_a is not None
65 assert lora.lora_b is not None
66 assert lora.lora_a.device == torch.device(device)
@@ -430,7 +427,7 @@ def test_lru_cache_worker_adapter_manager(dist_init, dummy_model, device, tmp_pa
427 vllm_config.scheduler_config.max_num_seqs = 4
428 vllm_config.scheduler_config.max_num_batched_tokens = 2
429 worker_adapter_manager = LRUCacheWorkerLoRAManager(
433 - vllm_config, device, EMBEDDING_MODULES, EMBEDDING_PADDING_MODULES
430 + vllm_config, device, EMBEDDING_MODULES
431 )
432
433 worker_adapter_manager.max_num_seqs = 4
@@ -533,9 +530,7 @@ def test_worker_adapter_manager(dist_init, dummy_model_gate_up, device, tmp_path
530 vllm_config.scheduler_config.max_num_seqs = 4
531 vllm_config.scheduler_config.max_num_batched_tokens = 2
532
536 - worker_adapter_manager = WorkerLoRAManager(
537 - vllm_config, device, EMBEDDING_MODULES, EMBEDDING_PADDING_MODULES
538 - )
533 + worker_adapter_manager = WorkerLoRAManager(vllm_config, device, EMBEDDING_MODULES)
534 worker_adapter_manager.vocab_size = dummy_model_gate_up.unpadded_vocab_size
535 worker_adapter_manager.create_lora_manager(dummy_model_gate_up)
536
tests/lora/test_olmoe_tp.py
+17 -5
@@ -40,7 +40,10 @@ EXPECTED_BASE_MODEL_OUTPUT = [
40
41
42 def generate_and_test(
43 - llm: vllm.LLM, lora_path: str, lora_id: list[int | None] | int | None
43 + llm: vllm.LLM,
44 + lora_path: str,
45 + lora_id: list[int | None] | int | None,
46 + compare_lower: bool = False,
47 ) -> None:
48 prompts = [
49 PROMPT_TEMPLATE.format(context="How many candidates are there?"),
@@ -74,12 +77,18 @@ def generate_and_test(
77
78 for i in range(len(EXPECTED_LORA_OUTPUT)):
79 req_lora_id = lora_id[i] if isinstance(lora_id, list) else lora_id
80 + generated_text = generated_texts[i]
81 expected_output = (
82 EXPECTED_LORA_OUTPUT[i]
83 if req_lora_id is not None
84 else EXPECTED_BASE_MODEL_OUTPUT[i]
85 )
82 - assert generated_texts[i].startswith(expected_output)
86 +
87 + if compare_lower:
88 + generated_text = generated_text.lower()
89 + expected_output = expected_output.lower()
90 +
91 + assert generated_text.startswith(expected_output)
92
93
94 def test_olmoe_lora(olmoe_lora_files):
@@ -146,6 +155,9 @@ def test_olmoe_lora_tp4(olmoe_lora_files, fully_sharded_loras):
155 tensor_parallel_size=4,
156 fully_sharded_loras=fully_sharded_loras,
157 )
149 -
150 - generate_and_test(llm, olmoe_lora_files, lora_id=1)
151 - generate_and_test(llm, olmoe_lora_files, lora_id=2)
158 + generate_and_test(
159 + llm, olmoe_lora_files, lora_id=1, compare_lower=fully_sharded_loras
160 + )
161 + generate_and_test(
162 + llm, olmoe_lora_files, lora_id=2, compare_lower=fully_sharded_loras
163 + )
tests/lora/test_peft_helper.py
+15 -13
@@ -25,31 +25,33 @@ ERROR_CASES = [
25 ]
26
27
28 -def test_peft_helper_pass(sql_lora_files, tmp_path):
28 +def test_peft_helper_pass(llama32_lora_files, tmp_path):
29 peft_helper = PEFTHelper.from_local_dir(
30 - sql_lora_files, max_position_embeddings=4096
30 + llama32_lora_files, max_position_embeddings=4096
31 )
32 lora_config = LoRAConfig(max_lora_rank=16, max_cpu_loras=3, max_loras=2)
33 peft_helper.validate_legal(lora_config)
34 assert peft_helper.r == 8
35 - assert peft_helper.lora_alpha == 16
36 - assert peft_helper.target_modules == [
37 - "q_proj",
38 - "v_proj",
39 - "k_proj",
40 - "o_proj",
41 - "gate_proj",
42 - "up_proj",
35 + assert peft_helper.lora_alpha == 32
36 + target_modules = sorted(peft_helper.target_modules)
37 +
38 + assert target_modules == [
39 "down_proj",
40 "embed_tokens",
41 + "gate_proj",
42 + "k_proj",
43 "lm_head",
44 + "o_proj",
45 + "q_proj",
46 + "up_proj",
47 + "v_proj",
48 ]
49 assert peft_helper.vllm_max_position_embeddings == 4096
50
51 # test RSLoRA
52 rslora_config = dict(use_rslora=True)
53 test_dir = tmp_path / "test_rslora"
52 - shutil.copytree(sql_lora_files, test_dir)
54 + shutil.copytree(llama32_lora_files, test_dir)
55
56 # Load and modify configuration
57 config_path = test_dir / "adapter_config.json"
@@ -70,14 +72,14 @@ def test_peft_helper_pass(sql_lora_files, tmp_path):
72
73 @pytest.mark.parametrize("test_name,config_change,expected_error", ERROR_CASES)
74 def test_peft_helper_error(
73 - sql_lora_files,
75 + llama32_lora_files,
76 tmp_path,
77 test_name: str,
78 config_change: dict,
79 expected_error: str,
80 ):
81 test_dir = tmp_path / test_name
80 - shutil.copytree(sql_lora_files, test_dir)
82 + shutil.copytree(llama32_lora_files, test_dir)
83
84 # Load and modify configuration
85 config_path = test_dir / "adapter_config.json"
tests/plugins/lora_resolvers/test_filesystem_resolver.py
+5 -5
@@ -8,8 +8,8 @@ from huggingface_hub import snapshot_download
8
9 from vllm.plugins.lora_resolvers.filesystem_resolver import FilesystemResolver
10
11 -MODEL_NAME = "mistralai/Mistral-7B-v0.1"
12 -LORA_NAME = "typeof/zephyr-7b-beta-lora"
11 +MODEL_NAME = "Qwen/Qwen3-0.6B"
12 +LORA_NAME = "charent/self_cognition_Alice"
13 PA_NAME = "swapnilbp/llama_tweet_ptune"
14
15
@@ -21,7 +21,7 @@ def adapter_cache(request, tmpdir_factory):
21
22
23 @pytest.fixture(scope="module")
24 -def zephyr_lora_files():
24 +def qwen3_lora_files():
25 return snapshot_download(repo_id=LORA_NAME)
26
27
@@ -31,9 +31,9 @@ def pa_files():
31
32
33 @pytest.mark.asyncio
34 -async def test_filesystem_resolver(adapter_cache, zephyr_lora_files):
34 +async def test_filesystem_resolver(adapter_cache, qwen3_lora_files):
35 model_files = adapter_cache / LORA_NAME
36 - shutil.copytree(zephyr_lora_files, model_files)
36 + shutil.copytree(qwen3_lora_files, model_files)
37
38 fs_resolver = FilesystemResolver(adapter_cache)
39 assert fs_resolver is not None
vllm/lora/models.py
+13 -21
@@ -103,7 +103,6 @@ class LoRAModel:
103 def check_lora_name(self, lora_name: str) -> bool:
104 return lora_name in self.loras
105
106 - # (yard1): TODO see if we can derive target_embedding_padding automatically
106 @classmethod
107 def from_lora_tensors(
108 cls,
@@ -112,9 +111,7 @@ class LoRAModel:
111 peft_helper: PEFTHelper,
112 device: str = "cuda",
113 dtype: torch.dtype | None = None,
115 - target_embedding_padding: int | None = None,
116 - embedding_modules: dict[str, str] | None = None,
117 - embedding_padding_modules: list[str] | None = None,
114 + model_vocab_size: int | None = None,
115 weights_mapper: WeightsMapper | None = None,
116 ) -> "LoRAModel":
117 """Create a LoRAModel from a dictionary of tensors."""
@@ -132,22 +129,21 @@ class LoRAModel:
129 )
130
131 if is_lora_a:
132 + if (
133 + "lora_embedding_A" in tensor_name
134 + and model_vocab_size is not None
135 + and model_vocab_size != tensor.shape[1]
136 + ):
137 + raise RuntimeError(
138 + f"The embedding LoRA size({tensor.shape[1]}) must be consistent"
139 + f" with the base model's vocabulary size({model_vocab_size})."
140 + )
141 loras[module_name].lora_a = tensor.to(device=device, dtype=dtype)
142 if pin_memory:
143 loras[module_name].lora_a = loras[module_name].lora_a.pin_memory()
144 else:
145 loras[module_name].lora_b = tensor.to(device=device, dtype=dtype)
140 - assert embedding_padding_modules is not None
141 - if (
142 - any(name in module_name for name in embedding_padding_modules)
143 - and target_embedding_padding is not None
144 - ):
145 - lora_b = loras[module_name].lora_b
146 - assert target_embedding_padding >= lora_b.shape[0]
147 - addition = target_embedding_padding - lora_b.shape[0]
148 - loras[module_name].lora_b = torch.nn.functional.pad(
149 - lora_b, (0, 0, 0, addition)
150 - )
146 +
147 if pin_memory:
148 loras[module_name].lora_b = loras[module_name].lora_b.pin_memory()
149
@@ -163,9 +159,7 @@ class LoRAModel:
159 lora_model_id: int | None = None,
160 device: str = "cuda",
161 dtype: torch.dtype | None = None,
166 - target_embedding_padding: int | None = None,
167 - embedding_modules: dict[str, str] | None = None,
168 - embedding_padding_modules: list[str] | None = None,
162 + model_vocab_size: int | None = None,
163 weights_mapper: WeightsMapper | None = None,
164 tensorizer_config_dict: dict | None = None,
165 ) -> "LoRAModel":
@@ -287,9 +281,7 @@ class LoRAModel:
281 peft_helper=peft_helper,
282 device=device,
283 dtype=dtype,
290 - target_embedding_padding=target_embedding_padding,
291 - embedding_modules=embedding_modules,
292 - embedding_padding_modules=embedding_padding_modules,
284 + model_vocab_size=model_vocab_size,
285 weights_mapper=weights_mapper,
286 )
287
vllm/lora/worker_manager.py
+1 -5
@@ -34,12 +34,10 @@ class WorkerLoRAManager:
34 vllm_config: VllmConfig,
35 device: torch.device,
36 embedding_modules: dict[str, str],
37 - embedding_padding_modules: list[str],
37 lora_model_cls: type[LoRAModel] = LoRAModel,
38 ):
39 self._lora_model_cls = lora_model_cls
40 self.embedding_modules = embedding_modules
42 - self.embedding_padding_modules = embedding_padding_modules
41 self._cached_dummy_lora: None | Literal[False] | LoRAModel = False
42 self.max_num_seqs = vllm_config.scheduler_config.max_num_seqs
43 self.max_num_batched_tokens = (
@@ -121,9 +119,7 @@ class WorkerLoRAManager:
119 lora_model_id=lora_request.lora_int_id,
120 device="cpu",
121 dtype=self.lora_config.lora_dtype,
124 - target_embedding_padding=self.vocab_size,
125 - embedding_modules=self.embedding_modules,
126 - embedding_padding_modules=self.embedding_padding_modules,
122 + model_vocab_size=self.vocab_size,
123 tensorizer_config_dict=lora_request.tensorizer_config_dict,
124 weights_mapper=hf_to_vllm_mapper,
125 )
vllm/model_executor/models/apertus.py
-1
@@ -482,7 +482,6 @@ class ApertusForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
482 "embed_tokens": "input_embeddings",
483 "lm_head": "output_embeddings",
484 }
485 - embedding_padding_modules = ["lm_head"]
485
486 def __init__(
487 self,
vllm/model_executor/models/bamba.py
-1
@@ -419,7 +419,6 @@ class BambaForCausalLM(
419 "embed_tokens": "input_embeddings",
420 "lm_head": "output_embeddings",
421 }
422 - embedding_padding_modules = ["lm_head"]
422
423 @classmethod
424 def get_mamba_state_dtype_from_config(
vllm/model_executor/models/exaone.py
-1
@@ -457,7 +457,6 @@ class ExaoneForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
457 "wte": "input_embeddings",
458 "lm_head": "output_embeddings",
459 }
460 - embedding_padding_modules = ["lm_head"]
460
461 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
462 super().__init__()
vllm/model_executor/models/exaone4.py
-1
@@ -450,7 +450,6 @@ class Exaone4ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
450 "embed_tokens": "input_embeddings",
451 "lm_head": "output_embeddings",
452 }
453 - embedding_padding_modules = ["lm_head"]
453
454 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
455 super().__init__()
vllm/model_executor/models/falcon_h1.py
-1
@@ -510,7 +510,6 @@ class FalconH1ForCausalLM(
510 "embed_tokens": "input_embeddings",
511 "lm_head": "output_embeddings",
512 }
513 - embedding_padding_modules = ["lm_head"]
513
514 @classmethod
515 def get_mamba_state_dtype_from_config(
vllm/model_executor/models/granite.py
-1
@@ -400,7 +400,6 @@ class GraniteForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
400 "embed_tokens": "input_embeddings",
401 "lm_head": "output_embeddings",
402 }
403 - embedding_padding_modules = ["lm_head"]
403
404 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
405 super().__init__()
vllm/model_executor/models/granitemoe.py
-1
@@ -497,7 +497,6 @@ class GraniteMoeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
497 "embed_tokens": "input_embeddings",
498 "lm_head": "output_embeddings",
499 }
500 - embedding_padding_modules = ["lm_head"]
500
501 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
502 super().__init__()
vllm/model_executor/models/granitemoehybrid.py
-1
@@ -601,7 +601,6 @@ class GraniteMoeHybridForCausalLM(
601 "embed_tokens": "input_embeddings",
602 "lm_head": "output_embeddings",
603 }
604 - embedding_padding_modules = ["lm_head"]
604
605 @classmethod
606 def get_mamba_state_dtype_from_config(
vllm/model_executor/models/granitemoeshared.py
-1
@@ -263,7 +263,6 @@ class GraniteMoeSharedForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
263 "embed_tokens": "input_embeddings",
264 "lm_head": "output_embeddings",
265 }
266 - embedding_padding_modules = ["lm_head"]
266
267 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
268 super().__init__()
vllm/model_executor/models/interfaces.py
-3
@@ -347,7 +347,6 @@ class SupportsLoRA(Protocol):
347 # The `embedding_module` and `embedding_padding_modules`
348 # are empty by default.
349 embedding_modules: ClassVar[dict[str, str]] = {}
350 - embedding_padding_modules: ClassVar[list[str]] = []
350 packed_modules_mapping: dict[str, list[str]] = {}
351
352
@@ -359,7 +358,6 @@ class _SupportsLoRAType(Protocol):
358
359 packed_modules_mapping: dict[str, list[str]]
360 embedding_modules: dict[str, str]
362 - embedding_padding_modules: list[str]
361
362
363 @overload
@@ -379,7 +377,6 @@ def supports_lora(
377 lora_attrs = (
378 "packed_modules_mapping",
379 "embedding_modules",
382 - "embedding_padding_modules",
380 )
381 missing_attrs = tuple(attr for attr in lora_attrs if not hasattr(model, attr))
382
vllm/model_executor/models/jamba.py
-1
@@ -480,7 +480,6 @@ class JambaForCausalLM(
480 "embed_tokens": "input_embeddings",
481 "lm_head": "output_embeddings",
482 }
483 - embedding_padding_modules = ["lm_head"]
483
484 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
485 config = vllm_config.model_config.hf_config
vllm/model_executor/models/lfm2.py
-1
@@ -422,7 +422,6 @@ class Lfm2ForCausalLM(
422 "embed_tokens": "input_embeddings",
423 "lm_head": "output_embeddings",
424 }
425 - embedding_padding_modules = ["lm_head"]
425
426 @classmethod
427 def get_mamba_state_dtype_from_config(
vllm/model_executor/models/lfm2_moe.py
-1
@@ -602,7 +602,6 @@ class Lfm2MoeForCausalLM(
602 "embed_tokens": "input_embeddings",
603 "lm_head": "output_embeddings",
604 }
605 - embedding_padding_modules = ["lm_head"]
605
606 @classmethod
607 def get_mamba_state_dtype_from_config(
vllm/model_executor/models/llama.py
-1
@@ -528,7 +528,6 @@ class LlamaForCausalLM(
528 "embed_tokens": "input_embeddings",
529 "lm_head": "output_embeddings",
530 }
531 - embedding_padding_modules = ["lm_head"]
531
532 # Mistral/Llama models can also be loaded with --load-format mistral
533 # from consolidated.safetensors checkpoints
vllm/model_executor/models/minicpm.py
-1
@@ -568,7 +568,6 @@ class MiniCPMForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsEagle3):
568 "embed_tokens": "input_embeddings",
569 "lm_head": "output_embeddings",
570 }
571 - embedding_padding_modules = ["lm_head"]
571
572 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
573 super().__init__()
vllm/model_executor/models/minicpm_eagle.py
-1
@@ -305,7 +305,6 @@ class EagleMiniCPMForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsEagle
305 "embed_tokens": "input_embeddings",
306 "lm_head": "output_embeddings",
307 }
308 - embedding_padding_modules = ["lm_head"]
308
309 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
310 super().__init__()
vllm/model_executor/models/minicpmv.py
-1
@@ -1741,5 +1741,4 @@ class MiniCPMV(MiniCPMVBaseModel, SupportsMultiModal, SupportsLoRA):
1741 # so update values before init is called
1742 cls.packed_modules_mapping.update(instance_cls.packed_modules_mapping)
1743 cls.embedding_modules.update(instance_cls.embedding_modules)
1744 - cls.embedding_padding_modules += instance_cls.embedding_padding_modules
1744 return instance_cls(vllm_config=vllm_config, prefix=prefix)
vllm/model_executor/models/mixtral.py
-1
@@ -496,7 +496,6 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA, SupportsPP, MixtureOfExperts):
496 "embed_tokens": "input_embeddings",
497 "lm_head": "output_embeddings",
498 }
499 - embedding_padding_modules = ["lm_head"]
499
500 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
501 super().__init__()
vllm/model_executor/models/nemotron.py
-1
@@ -439,7 +439,6 @@ class NemotronForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
439 "embed_tokens": "input_embeddings",
440 "lm_head": "output_embeddings",
441 }
442 - embedding_padding_modules = ["lm_head"]
442
443 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
444 super().__init__()
vllm/model_executor/models/nemotron_h.py
-1
@@ -713,7 +713,6 @@ class NemotronHForCausalLM(
713 "embed_tokens": "input_embeddings",
714 "lm_head": "output_embeddings",
715 }
716 - embedding_padding_modules = ["lm_head"]
716
717 @classmethod
718 def get_mamba_state_dtype_from_config(
vllm/model_executor/models/nemotron_nas.py
-1
@@ -387,7 +387,6 @@ class DeciLMForCausalLM(nn.Module, SupportsLoRA, SupportsPP, HasNoOps):
387 "embed_tokens": "input_embeddings",
388 "lm_head": "output_embeddings",
389 }
390 - embedding_padding_modules = ["lm_head"]
390
391 # Mistral/Llama models can also be loaded with --load-format mistral
392 # from consolidated.safetensors checkpoints
vllm/model_executor/models/phimoe.py
-1
@@ -617,7 +617,6 @@ class PhiMoEForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
617 "embed_tokens": "input_embeddings",
618 "lm_head": "output_embeddings",
619 }
620 - embedding_padding_modules = ["lm_head"]
620
621 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
622 super().__init__()
vllm/model_executor/models/solar.py
-1
@@ -426,7 +426,6 @@ class SolarForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
426 "embed_tokens": "input_embeddings",
427 "lm_head": "output_embeddings",
428 }
429 - embedding_padding_modules = ["lm_head"]
429
430 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
431 super().__init__()
vllm/model_executor/models/transformers/base.py
-1
@@ -93,7 +93,6 @@ ALL_ATTENTION_FUNCTIONS["vllm"] = vllm_flash_attention_forward
93
94
95 class Base(nn.Module, VllmModel, SupportsQuant, SupportsLoRA, SupportsPP):
96 - embedding_padding_modules = ["lm_head"]
96 embedding_modules = ["embed_tokens"] # TODO transformers will have a util to get it
97 hf_to_vllm_mapper = WeightsMapper(
98 orig_to_new_prefix={
vllm/v1/worker/lora_model_runner_mixin.py
-1
@@ -43,7 +43,6 @@ class LoRAModelRunnerMixin:
43 vllm_config,
44 device,
45 model.embedding_modules,
46 - model.embedding_padding_modules,
46 )
47 return self.lora_manager.create_lora_manager(model)
48