A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 05:45 UTC
[CI/Build][LoRA] Update Qwen35 LoRA testing (#37816)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Mar 23, 2026 at 12:55 UTC
1f0d21064137f4fdc816864cf01832c26350be5a
4 files changed
+369
-135
.buildkite/test_areas/lora.yaml
+2
-2
@@ -8,7 +8,7 @@ steps:
8
- vllm/lora
9
- tests/lora
10
commands:
11
- - pytest -v -s lora --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --ignore=lora/test_chatglm3_tp.py --ignore=lora/test_llama_tp.py --ignore=lora/test_llm_with_multi_loras.py --ignore=lora/test_olmoe_tp.py --ignore=lora/test_deepseekv2_tp.py --ignore=lora/test_gptoss_tp.py --ignore=lora/test_qwen3moe_tp.py --ignore=lora/test_qwen35_densemoel_lora.py
11
+ - pytest -v -s lora --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --ignore=lora/test_chatglm3_tp.py --ignore=lora/test_llama_tp.py --ignore=lora/test_llm_with_multi_loras.py --ignore=lora/test_olmoe_tp.py --ignore=lora/test_deepseekv2_tp.py --ignore=lora/test_gptoss_tp.py --ignore=lora/test_qwen3moe_tp.py --ignore=lora/test_qwen35_densemodel_lora.py
12
parallelism: 4
13
14
@@ -31,4 +31,4 @@ steps:
31
- pytest -v -s -x lora/test_llm_with_multi_loras.py
32
- pytest -v -s -x lora/test_olmoe_tp.py
33
- pytest -v -s -x lora/test_gptoss_tp.py
34
- - pytest -v -s -x lora/test_qwen35_densemoel_lora.py
\ No newline at end of file
34
+ - pytest -v -s -x lora/test_qwen35_densemodel_lora.py
\ No newline at end of file
tests/lora/conftest.py
+6
-1
@@ -295,10 +295,15 @@ def whisper_lora_files():
295
296
297
@pytest.fixture(scope="session")
298
-def qwen35_dense_model_lora_files():
298
+def qwen35_text_lora_files():
299
return snapshot_download(repo_id="jeeejeee/qwen35-4b-text-only-sql-lora")
300
301
302
+@pytest.fixture(scope="session")
303
+def qwen35_vl_lora_files():
304
+ return snapshot_download(repo_id="jeeejeee/qwen35-4b-all-linear-pokemon-lora")
305
+
306
+
307
@pytest.fixture
308
def reset_default_device():
309
"""
tests/lora/test_qwen35_densemodel_lora.py
new
+361
@@ -0,0 +1,361 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+from transformers import AutoTokenizer
5
+
6
+import vllm
7
+import vllm.config
8
+from vllm.assets.image import ImageAsset
9
+from vllm.lora.request import LoRARequest
10
+
11
+from ..utils import create_new_process_for_each_test, multi_gpu_test
12
+
13
+MODEL_PATH = "Qwen/Qwen3.5-4B"
14
+TEXT_LORA_ID = 1
15
+VL_LORA_ID = 2
16
+
17
+# text-only task
18
+TEXT_PROMPT_TEMPLATE = """Write a SQL query for the given database.\nSchema:\nTables:\n - stadium(Stadium_ID, Location, Name, Capacity, Highest, Lowest, Average)\n - singer(Singer_ID, Name, Country, Song_Name, Song_release_year, Age, Is_male)\n - concert(concert_ID, concert_Name, Theme, Stadium_ID, Year)\n - singer_in_concert(concert_ID, Singer_ID)\n\nQuestion:\n{query}""" # noqa: E501
19
+
20
+TEXT_EXPECTED_LORA_OUTPUT = [
21
+ "SELECT count(*) FROM singer",
22
+ "SELECT avg(age) , min(age) , max(age) FROM singer WHERE country = 'France'",
23
+ "SELECT name FROM stadium WHERE stadium_id NOT IN (SELECT stadium_id FROM concert)",
24
+]
25
+
26
+
27
+# visual caption
28
+VL_QUESTION = "What is in the image?"
29
+VL_TEST_IMAGES = [
30
+ ImageAsset("stop_sign"),
31
+ ImageAsset("cherry_blossom"),
32
+]
33
+VL_EXPECTED_LORA_OUTPUT = [
34
+ 'A red STOP sign stands prominently in the foreground, with a traditional Chinese gate adorned with red lanterns and the Chinese characters "中華門" in the background, signaling the entrance to a Chinatown. A black car passes by on the street, and stone lion statues guard the entrance to the culturally rich area.', # noqa: E501
35
+ "A vibrant blue sky serves as a backdrop for the iconic Tokyo Skytree, partially obscured by the delicate pink blossoms of cherry trees in full bloom.", # noqa: E501
36
+]
37
+
38
+TOKENIZER = AutoTokenizer.from_pretrained(MODEL_PATH, trust_remote_code=True)
39
+
40
+
41
+def _assert_exact_outputs(
42
+ generated_texts: list[str], expected_outputs: list[str]
43
+) -> None:
44
+ assert generated_texts == expected_outputs
45
+
46
+
47
+def _assert_prefix_outputs(
48
+ generated_texts: list[str],
49
+ expected_outputs: list[str],
50
+) -> None:
51
+ assert len(generated_texts) == len(expected_outputs)
52
+ for generated_text, expected_text in zip(generated_texts, expected_outputs):
53
+ assert expected_text.startswith(generated_text), (
54
+ f"Generated {generated_text!r} is not a prefix of expected "
55
+ f"{expected_text!r}"
56
+ )
57
+
58
+
59
+def _run_text_lora_sample(
60
+ llm: vllm.LLM,
61
+ lora_path: str,
62
+ lora_id: int,
63
+) -> list[str]:
64
+ prompts = [
65
+ TEXT_PROMPT_TEMPLATE.format(query="How many singers do we have?"),
66
+ TEXT_PROMPT_TEMPLATE.format(
67
+ query=(
68
+ "What is the average, minimum, and maximum "
69
+ "age of all singers from France?"
70
+ )
71
+ ),
72
+ TEXT_PROMPT_TEMPLATE.format(
73
+ query="What are the names of the stadiums without any concerts?"
74
+ ),
75
+ ]
76
+ input_templates = []
77
+ for prompt_text in prompts:
78
+ messages = [{"role": "user", "content": prompt_text}]
79
+ prompt = TOKENIZER.apply_chat_template(
80
+ messages,
81
+ tokenize=False,
82
+ add_generation_prompt=True,
83
+ enable_thinking=False, # disable thinking
84
+ )
85
+ input_templates.append(prompt)
86
+
87
+ outputs = llm.generate(
88
+ input_templates,
89
+ vllm.SamplingParams(temperature=0, max_tokens=512),
90
+ lora_request=LoRARequest(str(lora_id), lora_id, lora_path),
91
+ )
92
+
93
+ generated_texts: list[str] = []
94
+ for output in outputs:
95
+ generated_text = output.outputs[0].text.strip()
96
+ generated_texts.append(generated_text)
97
+ print(f"Prompt: {output.prompt!r}, Generated text: {generated_text!r}")
98
+ return generated_texts
99
+
100
+
101
+def _run_vl_lora_sample(
102
+ llm: vllm.LLM,
103
+ lora_path: str | None = None,
104
+ lora_id: int = 0,
105
+) -> list[str]:
106
+ messages = [
107
+ {
108
+ "role": "user",
109
+ "content": [
110
+ {"type": "image"},
111
+ {"type": "text", "text": VL_QUESTION},
112
+ ],
113
+ }
114
+ ]
115
+ prompt = TOKENIZER.apply_chat_template(
116
+ messages,
117
+ tokenize=False,
118
+ add_generation_prompt=True,
119
+ enable_thinking=False,
120
+ )
121
+ prompts = [
122
+ {
123
+ "prompt": prompt,
124
+ "multi_modal_data": {"image": asset.pil_image},
125
+ }
126
+ for asset in VL_TEST_IMAGES
127
+ ]
128
+ outputs = llm.generate(
129
+ prompts,
130
+ vllm.SamplingParams(temperature=0, max_tokens=128),
131
+ lora_request=(
132
+ LoRARequest(str(lora_id), lora_id, lora_path)
133
+ if lora_path is not None
134
+ else None
135
+ ),
136
+ )
137
+
138
+ generated_texts: list[str] = []
139
+ for output in outputs:
140
+ generated_text = output.outputs[0].text.strip()
141
+ generated_texts.append(generated_text)
142
+ print(f"Prompt: {output.prompt!r}, Generated text: {generated_text!r}")
143
+ return generated_texts
144
+
145
+
146
+def _build_text_prompts() -> list[str]:
147
+ prompts = [
148
+ TEXT_PROMPT_TEMPLATE.format(query="How many singers do we have?"),
149
+ TEXT_PROMPT_TEMPLATE.format(
150
+ query=(
151
+ "What is the average, minimum, and maximum "
152
+ "age of all singers from France?"
153
+ )
154
+ ),
155
+ TEXT_PROMPT_TEMPLATE.format(
156
+ query="What are the names of the stadiums without any concerts?"
157
+ ),
158
+ ]
159
+ input_templates = []
160
+ for prompt_text in prompts:
161
+ messages = [{"role": "user", "content": prompt_text}]
162
+ prompt = TOKENIZER.apply_chat_template(
163
+ messages,
164
+ tokenize=False,
165
+ add_generation_prompt=True,
166
+ enable_thinking=False,
167
+ )
168
+ input_templates.append(prompt)
169
+ return input_templates
170
+
171
+
172
+def _build_vl_prompts() -> list[dict]:
173
+ messages = [
174
+ {
175
+ "role": "user",
176
+ "content": [
177
+ {"type": "image"},
178
+ {"type": "text", "text": VL_QUESTION},
179
+ ],
180
+ }
181
+ ]
182
+ prompt = TOKENIZER.apply_chat_template(
183
+ messages,
184
+ tokenize=False,
185
+ add_generation_prompt=True,
186
+ enable_thinking=False,
187
+ )
188
+ return [
189
+ {
190
+ "prompt": prompt,
191
+ "multi_modal_data": {"image": asset.pil_image},
192
+ }
193
+ for asset in VL_TEST_IMAGES
194
+ ]
195
+
196
+
197
+def _run_mixed_lora_sample(
198
+ llm: vllm.LLM,
199
+ text_lora_path: str,
200
+ vl_lora_path: str,
201
+ text_lora_id: int,
202
+ vl_lora_id: int,
203
+) -> list[str]:
204
+ text_prompts = _build_text_prompts()[:2]
205
+ vl_prompts = _build_vl_prompts()
206
+ prompts = [
207
+ text_prompts[0],
208
+ vl_prompts[0],
209
+ text_prompts[1],
210
+ vl_prompts[1],
211
+ ]
212
+ lora_requests = [
213
+ LoRARequest("qwen35-text", text_lora_id, text_lora_path),
214
+ LoRARequest("qwen35-vl", vl_lora_id, vl_lora_path),
215
+ LoRARequest("qwen35-text", text_lora_id, text_lora_path),
216
+ LoRARequest("qwen35-vl", vl_lora_id, vl_lora_path),
217
+ ]
218
+ outputs = llm.generate(
219
+ prompts,
220
+ vllm.SamplingParams(temperature=0, max_tokens=256),
221
+ lora_request=lora_requests,
222
+ )
223
+
224
+ generated_texts: list[str] = []
225
+ for output in outputs:
226
+ generated_text = output.outputs[0].text.strip()
227
+ generated_texts.append(generated_text)
228
+ print(f"Prompt: {output.prompt!r}, Generated text: {generated_text!r}")
229
+ return generated_texts
230
+
231
+
232
+def _run_mixed_lora_and_base_sample(
233
+ llm: vllm.LLM,
234
+ text_lora_path: str,
235
+ vl_lora_path: str,
236
+ text_lora_id: int,
237
+ vl_lora_id: int,
238
+) -> list[str]:
239
+ text_prompt = _build_text_prompts()[0]
240
+ vl_prompt = _build_vl_prompts()[0]
241
+ prompts = [
242
+ text_prompt,
243
+ vl_prompt,
244
+ text_prompt,
245
+ vl_prompt,
246
+ ]
247
+ lora_requests = [
248
+ LoRARequest("qwen35-text", text_lora_id, text_lora_path),
249
+ LoRARequest("qwen35-vl", vl_lora_id, vl_lora_path),
250
+ None,
251
+ None,
252
+ ]
253
+ outputs = llm.generate(
254
+ prompts,
255
+ vllm.SamplingParams(temperature=0, max_tokens=256),
256
+ lora_request=lora_requests,
257
+ )
258
+
259
+ generated_texts: list[str] = []
260
+ for output in outputs:
261
+ generated_text = output.outputs[0].text.strip()
262
+ generated_texts.append(generated_text)
263
+ print(f"Prompt: {output.prompt!r}, Generated text: {generated_text!r}")
264
+ return generated_texts
265
+
266
+
267
+def _assert_qwen35_text_vl_and_mixed_lora(
268
+ llm: vllm.LLM,
269
+ qwen35_text_lora_files: str,
270
+ qwen35_vl_lora_files: str,
271
+) -> None:
272
+ generated_texts = _run_text_lora_sample(
273
+ llm,
274
+ qwen35_text_lora_files,
275
+ TEXT_LORA_ID,
276
+ )
277
+
278
+ _assert_exact_outputs(generated_texts, TEXT_EXPECTED_LORA_OUTPUT)
279
+
280
+ generated_texts = _run_vl_lora_sample(
281
+ llm,
282
+ qwen35_vl_lora_files,
283
+ VL_LORA_ID,
284
+ )
285
+ _assert_prefix_outputs(generated_texts, VL_EXPECTED_LORA_OUTPUT)
286
+
287
+ generated_texts = _run_mixed_lora_sample(
288
+ llm,
289
+ qwen35_text_lora_files,
290
+ qwen35_vl_lora_files,
291
+ text_lora_id=TEXT_LORA_ID,
292
+ vl_lora_id=VL_LORA_ID,
293
+ )
294
+ assert generated_texts[0] == TEXT_EXPECTED_LORA_OUTPUT[0]
295
+ assert generated_texts[2] == TEXT_EXPECTED_LORA_OUTPUT[1]
296
+ _assert_prefix_outputs([generated_texts[1]], [VL_EXPECTED_LORA_OUTPUT[0]])
297
+ _assert_prefix_outputs([generated_texts[3]], [VL_EXPECTED_LORA_OUTPUT[1]])
298
+
299
+ generated_texts = _run_mixed_lora_and_base_sample(
300
+ llm,
301
+ qwen35_text_lora_files,
302
+ qwen35_vl_lora_files,
303
+ text_lora_id=TEXT_LORA_ID,
304
+ vl_lora_id=VL_LORA_ID,
305
+ )
306
+ assert generated_texts[0] == TEXT_EXPECTED_LORA_OUTPUT[0]
307
+ _assert_prefix_outputs([generated_texts[1]], [VL_EXPECTED_LORA_OUTPUT[0]])
308
+ assert generated_texts[2] != TEXT_EXPECTED_LORA_OUTPUT[0]
309
+ assert not VL_EXPECTED_LORA_OUTPUT[0].startswith(generated_texts[3]), (
310
+ "Non-LoRA vision output unexpectedly matches the LoRA expectation."
311
+ )
312
+
313
+
314
+@create_new_process_for_each_test()
315
+def test_qwen35_text_lora(qwen35_text_lora_files, qwen35_vl_lora_files):
316
+ llm = vllm.LLM(
317
+ model=MODEL_PATH,
318
+ max_model_len=4096,
319
+ enable_lora=True,
320
+ max_loras=2,
321
+ max_num_seqs=4,
322
+ max_lora_rank=8,
323
+ enforce_eager=True,
324
+ trust_remote_code=True,
325
+ enable_tower_connector_lora=True,
326
+ mm_processor_cache_gb=0,
327
+ limit_mm_per_prompt={"image": 1},
328
+ )
329
+
330
+ _assert_qwen35_text_vl_and_mixed_lora(
331
+ llm,
332
+ qwen35_text_lora_files,
333
+ qwen35_vl_lora_files,
334
+ )
335
+
336
+
337
+@multi_gpu_test(num_gpus=4)
338
+def test_qwen35_text_lora_tp4(qwen35_text_lora_files, qwen35_vl_lora_files):
339
+ llm = vllm.LLM(
340
+ model=MODEL_PATH,
341
+ max_model_len=4096,
342
+ enable_lora=True,
343
+ max_loras=2,
344
+ max_lora_rank=8,
345
+ max_num_seqs=4,
346
+ enforce_eager=True,
347
+ tensor_parallel_size=4,
348
+ trust_remote_code=True,
349
+ enable_tower_connector_lora=True,
350
+ mm_processor_cache_gb=0,
351
+ limit_mm_per_prompt={"image": 1},
352
+ compilation_config=vllm.config.CompilationConfig(
353
+ cudagraph_specialize_lora=False,
354
+ ),
355
+ )
356
+
357
+ _assert_qwen35_text_vl_and_mixed_lora(
358
+ llm,
359
+ qwen35_text_lora_files,
360
+ qwen35_vl_lora_files,
361
+ )
tests/lora/test_qwen35_densemoel_lora.py
deleted
-132
@@ -1,132 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-from transformers import AutoTokenizer
5
-
6
-import vllm
7
-import vllm.config
8
-from vllm.lora.request import LoRARequest
9
-
10
-from ..utils import create_new_process_for_each_test, multi_gpu_test
11
-
12
-MODEL_PATH = "Qwen/Qwen3.5-4B"
13
-
14
-PROMPT_TEMPLATE = """Write a SQL query for the given database.\nSchema:\nTables:\n - stadium(Stadium_ID, Location, Name, Capacity, Highest, Lowest, Average)\n - singer(Singer_ID, Name, Country, Song_Name, Song_release_year, Age, Is_male)\n - concert(concert_ID, concert_Name, Theme, Stadium_ID, Year)\n - singer_in_concert(concert_ID, Singer_ID)\n\nQuestion:\n{query}""" # noqa: E501
15
-
16
-EXPECTED_LORA_OUTPUT = [
17
- "SELECT count(*) FROM singer",
18
- "SELECT avg(age) , min(age) , max(age) FROM singer WHERE country = 'France'",
19
- "SELECT name FROM stadium WHERE stadium_id NOT IN (SELECT stadium_id FROM concert)",
20
-]
21
-
22
-
23
-tokenizer = AutoTokenizer.from_pretrained(MODEL_PATH, trust_remote_code=True)
24
-
25
-
26
-def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> list[str]:
27
- prompts = [
28
- PROMPT_TEMPLATE.format(query="How many singers do we have?"),
29
- PROMPT_TEMPLATE.format(
30
- query=(
31
- "What is the average, minimum, and maximum "
32
- "age of all singers from France?"
33
- )
34
- ),
35
- PROMPT_TEMPLATE.format(
36
- query=("What are the names of the stadiums without any concerts?")
37
- ),
38
- ]
39
- input_templates = []
40
- for prmpt in prompts:
41
- messages = [{"role": "user", "content": prmpt}]
42
- prompt = tokenizer.apply_chat_template(
43
- messages,
44
- tokenize=False,
45
- add_generation_prompt=True,
46
- enable_thinking=False, # disable thinking
47
- )
48
- input_templates.append(prompt)
49
- sampling_params = vllm.SamplingParams(temperature=0, max_tokens=512)
50
- outputs = llm.generate(
51
- input_templates,
52
- sampling_params,
53
- lora_request=LoRARequest(str(lora_id), lora_id, lora_path) if lora_id else None,
54
- )
55
-
56
- generated_texts: list[str] = []
57
- for output in outputs:
58
- prompt = output.prompt
59
- generated_text = output.outputs[0].text.strip()
60
- generated_texts.append(generated_text)
61
- print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
62
- return generated_texts
63
-
64
-
65
-@create_new_process_for_each_test()
66
-def test_qwen35_dense_model_lora(qwen35_dense_model_lora_files):
67
- llm = vllm.LLM(
68
- MODEL_PATH,
69
- max_model_len=512,
70
- enable_lora=True,
71
- max_loras=2,
72
- max_num_seqs=16,
73
- max_lora_rank=8,
74
- trust_remote_code=True,
75
- )
76
-
77
- output1 = do_sample(llm, qwen35_dense_model_lora_files, lora_id=1)
78
- for i in range(len(EXPECTED_LORA_OUTPUT)):
79
- assert output1[i] == EXPECTED_LORA_OUTPUT[i]
80
- output2 = do_sample(llm, qwen35_dense_model_lora_files, lora_id=2)
81
- for i in range(len(EXPECTED_LORA_OUTPUT)):
82
- assert output2[i] == EXPECTED_LORA_OUTPUT[i]
83
-
84
-
85
-@multi_gpu_test(num_gpus=4)
86
-def test_qwen35_dense_model_lora_tp4(qwen35_dense_model_lora_files):
87
- llm = vllm.LLM(
88
- MODEL_PATH,
89
- max_model_len=1024,
90
- enable_lora=True,
91
- max_loras=2,
92
- max_lora_rank=8,
93
- max_num_seqs=16,
94
- tensor_parallel_size=4,
95
- trust_remote_code=True,
96
- fully_sharded_loras=False,
97
- compilation_config=vllm.config.CompilationConfig( # Avoid OOM
98
- cudagraph_specialize_lora=False,
99
- ),
100
- )
101
-
102
- output1 = do_sample(llm, qwen35_dense_model_lora_files, lora_id=1)
103
- print(output1)
104
- for i in range(len(EXPECTED_LORA_OUTPUT)):
105
- assert output1[i] == EXPECTED_LORA_OUTPUT[i]
106
- output2 = do_sample(llm, qwen35_dense_model_lora_files, lora_id=2)
107
- for i in range(len(EXPECTED_LORA_OUTPUT)):
108
- assert output2[i] == EXPECTED_LORA_OUTPUT[i]
109
-
110
-
111
-@multi_gpu_test(num_gpus=4)
112
-def test_qwen35_dense_model_lora_tp4_fully_sharded_loras(qwen35_dense_model_lora_files):
113
- llm = vllm.LLM(
114
- MODEL_PATH,
115
- max_model_len=512,
116
- enable_lora=True,
117
- max_loras=2,
118
- max_lora_rank=8,
119
- tensor_parallel_size=4,
120
- trust_remote_code=True,
121
- fully_sharded_loras=True,
122
- gpu_memory_utilization=0.8,
123
- compilation_config=vllm.config.CompilationConfig( # Avoid OOM
124
- cudagraph_specialize_lora=False,
125
- ),
126
- )
127
- output1 = do_sample(llm, qwen35_dense_model_lora_files, lora_id=1)
128
- for i in range(len(EXPECTED_LORA_OUTPUT)):
129
- assert output1[i] == EXPECTED_LORA_OUTPUT[i]
130
- output2 = do_sample(llm, qwen35_dense_model_lora_files, lora_id=2)
131
- for i in range(len(EXPECTED_LORA_OUTPUT)):
132
- assert output2[i] == EXPECTED_LORA_OUTPUT[i]