| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import asyncio |
| 5 | import json |
| 6 | import shutil |
| 7 | from contextlib import suppress |
| 8 | |
| 9 | import openai # use the official client for correctness check |
| 10 | import pytest |
| 11 | import pytest_asyncio |
| 12 | |
| 13 | from tests.utils import RemoteOpenAIServer |
| 14 | |
| 15 | # any model with a chat template should work here |
| 16 | MODEL_NAME = "Qwen/Qwen3-0.6B" |
| 17 | |
| 18 | |
| 19 | BADREQUEST_CASES = [ |
| 20 | ( |
| 21 | "test_rank", |
| 22 | {"r": 1024}, |
| 23 | "is greater than max_lora_rank", |
| 24 | ), |
| 25 | ("test_dora", {"use_dora": True}, "does not yet support DoRA"), |
| 26 | ( |
| 27 | "test_modules_to_save", |
| 28 | {"modules_to_save": ["lm_head"]}, |
| 29 | "Unsupported modules_to_save", |
| 30 | ), |
| 31 | ] |
| 32 | |
| 33 | |
| 34 | @pytest.fixture(scope="module", params=[True]) |
| 35 | def server_with_lora_modules_json(request, qwen3_lora_files): |
| 36 | # Define the json format LoRA module configurations |
| 37 | lora_module_1 = { |
| 38 | "name": "qwen3-lora", |
| 39 | "path": qwen3_lora_files, |
| 40 | "base_model_name": MODEL_NAME, |
| 41 | } |
| 42 | |
| 43 | args = [ |
| 44 | # use half precision for speed and memory savings in CI environment |
| 45 | "--dtype", |
| 46 | "bfloat16", |
| 47 | "--max-model-len", |
| 48 | "8192", |
| 49 | "--enforce-eager", |
| 50 | # lora config below |
| 51 | "--enable-lora", |
| 52 | "--lora-modules", |
| 53 | json.dumps(lora_module_1), |
| 54 | "--max-lora-rank", |
| 55 | "64", |
| 56 | "--max-cpu-loras", |
| 57 | "2", |
| 58 | "--max-num-seqs", |
| 59 | "64", |
| 60 | ] |
| 61 | |
| 62 | # Enable the /v1/load_lora_adapter endpoint |
| 63 | envs = {"VLLM_ALLOW_RUNTIME_LORA_UPDATING": "True"} |
| 64 | |
| 65 | with RemoteOpenAIServer(MODEL_NAME, args, env_dict=envs) as remote_server: |
| 66 | yield remote_server |
| 67 | |
| 68 | |
| 69 | @pytest_asyncio.fixture |
| 70 | async def client(server_with_lora_modules_json): |
| 71 | async with server_with_lora_modules_json.get_async_client() as async_client: |
| 72 | yield async_client |
| 73 | |
| 74 | |
| 75 | @pytest.mark.asyncio |
| 76 | async def test_static_lora_lineage(client: openai.AsyncOpenAI, qwen3_lora_files): |
| 77 | models = await client.models.list() |
| 78 | models = models.data |
| 79 | served_model = models[0] |
| 80 | lora_models = models[1:] |
| 81 | assert served_model.id == MODEL_NAME |
| 82 | assert served_model.root == MODEL_NAME |
| 83 | assert served_model.parent is None |
| 84 | assert all(lora_model.root == qwen3_lora_files for lora_model in lora_models) |
| 85 | assert all(lora_model.parent == MODEL_NAME for lora_model in lora_models) |
| 86 | assert lora_models[0].id == "qwen3-lora" |
| 87 | |
| 88 | |
| 89 | @pytest.mark.asyncio |
| 90 | async def test_dynamic_lora_lineage(client: openai.AsyncOpenAI, qwen3_lora_files): |
| 91 | response = await client.post( |
| 92 | "load_lora_adapter", |
| 93 | cast_to=str, |
| 94 | body={"lora_name": "qwen3-lora-3", "lora_path": qwen3_lora_files}, |
| 95 | ) |
| 96 | # Ensure adapter loads before querying /models |
| 97 | assert "success" in response |
| 98 | |
| 99 | models = await client.models.list() |
| 100 | models = models.data |
| 101 | dynamic_lora_model = models[-1] |
| 102 | assert dynamic_lora_model.root == qwen3_lora_files |
| 103 | assert dynamic_lora_model.parent == MODEL_NAME |
| 104 | assert dynamic_lora_model.id == "qwen3-lora-3" |
| 105 | |
| 106 | |
| 107 | @pytest.mark.asyncio |
| 108 | async def test_load_lora_adapter_with_same_name_replaces_inplace( |
| 109 | client: openai.AsyncOpenAI, qwen3_meowing_lora_files, qwen3_woofing_lora_files |
| 110 | ): |
| 111 | """Test that loading a LoRA adapter with the same name replaces it inplace.""" |
| 112 | adapter_name = "replaceable-adapter" |
| 113 | messages = [ |
| 114 | {"content": "Follow the instructions to make animal noises", "role": "system"}, |
| 115 | {"content": "Make your favorite animal noise.", "role": "user"}, |
| 116 | ] |
| 117 | |
| 118 | # Load LoRA that makes model meow |
| 119 | response = await client.post( |
| 120 | "load_lora_adapter", |
| 121 | cast_to=str, |
| 122 | body={"lora_name": adapter_name, "lora_path": qwen3_meowing_lora_files}, |
| 123 | ) |
| 124 | assert "success" in response.lower() |
| 125 | |
| 126 | completion = await client.chat.completions.create( |
| 127 | model=adapter_name, |
| 128 | messages=messages, |
| 129 | max_tokens=10, |
| 130 | ) |
| 131 | assert "Meow Meow Meow" in completion.choices[0].message.content |
| 132 | |
| 133 | # Load LoRA that makes model woof |
| 134 | response = await client.post( |
| 135 | "load_lora_adapter", |
| 136 | cast_to=str, |
| 137 | body={ |
| 138 | "lora_name": adapter_name, |
| 139 | "lora_path": qwen3_woofing_lora_files, |
| 140 | "load_inplace": True, |
| 141 | }, |
| 142 | ) |
| 143 | assert "success" in response.lower() |
| 144 | |
| 145 | completion = await client.chat.completions.create( |
| 146 | model=adapter_name, |
| 147 | messages=messages, |
| 148 | max_tokens=10, |
| 149 | ) |
| 150 | assert "Woof Woof Woof" in completion.choices[0].message.content |
| 151 | |
| 152 | |
| 153 | @pytest.mark.asyncio |
| 154 | async def test_load_lora_adapter_with_load_inplace_false_errors( |
| 155 | client: openai.AsyncOpenAI, qwen3_meowing_lora_files |
| 156 | ): |
| 157 | """Test that load_inplace=False returns an error when adapter already exists.""" |
| 158 | adapter_name = "test-load-inplace-false" |
| 159 | |
| 160 | # Load LoRA adapter first time (should succeed) |
| 161 | response = await client.post( |
| 162 | "load_lora_adapter", |
| 163 | cast_to=str, |
| 164 | body={"lora_name": adapter_name, "lora_path": qwen3_meowing_lora_files}, |
| 165 | ) |
| 166 | assert "success" in response.lower() |
| 167 | |
| 168 | # Try to load the same adapter again with load_inplace=False (should fail) |
| 169 | with pytest.raises(openai.BadRequestError) as exc_info: |
| 170 | await client.post( |
| 171 | "load_lora_adapter", |
| 172 | cast_to=str, |
| 173 | body={ |
| 174 | "lora_name": adapter_name, |
| 175 | "lora_path": qwen3_meowing_lora_files, |
| 176 | }, |
| 177 | ) |
| 178 | |
| 179 | # Verify the error message |
| 180 | assert "already been loaded" in str(exc_info.value) |
| 181 | |
| 182 | |
| 183 | @pytest.mark.asyncio |
| 184 | async def test_dynamic_lora_not_found(client: openai.AsyncOpenAI): |
| 185 | with pytest.raises(openai.NotFoundError): |
| 186 | await client.post( |
| 187 | "load_lora_adapter", |
| 188 | cast_to=str, |
| 189 | body={"lora_name": "notfound", "lora_path": "/not/an/adapter"}, |
| 190 | ) |
| 191 | |
| 192 | |
| 193 | @pytest.mark.asyncio |
| 194 | async def test_dynamic_lora_invalid_files(client: openai.AsyncOpenAI, tmp_path): |
| 195 | invalid_files = tmp_path / "invalid_files" |
| 196 | invalid_files.mkdir() |
| 197 | (invalid_files / "adapter_config.json").write_text("this is not json") |
| 198 | |
| 199 | with pytest.raises(openai.InternalServerError): |
| 200 | await client.post( |
| 201 | "load_lora_adapter", |
| 202 | cast_to=str, |
| 203 | body={"lora_name": "invalid-json", "lora_path": str(invalid_files)}, |
| 204 | ) |
| 205 | |
| 206 | |
| 207 | @pytest.mark.asyncio |
| 208 | @pytest.mark.parametrize("test_name,config_change,expected_error", BADREQUEST_CASES) |
| 209 | async def test_dynamic_lora_badrequests( |
| 210 | client: openai.AsyncOpenAI, |
| 211 | tmp_path, |
| 212 | qwen3_lora_files, |
| 213 | test_name: str, |
| 214 | config_change: dict, |
| 215 | expected_error: str, |
| 216 | ): |
| 217 | # Create test directory |
| 218 | test_dir = tmp_path / test_name |
| 219 | |
| 220 | # Copy adapter files |
| 221 | shutil.copytree(qwen3_lora_files, test_dir) |
| 222 | |
| 223 | # Load and modify configuration |
| 224 | config_path = test_dir / "adapter_config.json" |
| 225 | with open(config_path) as f: |
| 226 | adapter_config = json.load(f) |
| 227 | # Apply configuration changes |
| 228 | adapter_config.update(config_change) |
| 229 | |
| 230 | # Save modified configuration |
| 231 | with open(config_path, "w") as f: |
| 232 | json.dump(adapter_config, f) |
| 233 | |
| 234 | # Test loading the adapter |
| 235 | with pytest.raises(openai.InternalServerError, match=expected_error): |
| 236 | await client.post( |
| 237 | "load_lora_adapter", |
| 238 | cast_to=str, |
| 239 | body={"lora_name": test_name, "lora_path": str(test_dir)}, |
| 240 | ) |
| 241 | |
| 242 | |
| 243 | @pytest.mark.asyncio |
| 244 | async def test_multiple_lora_adapters( |
| 245 | client: openai.AsyncOpenAI, tmp_path, qwen3_lora_files |
| 246 | ): |
| 247 | """Validate that many loras can be dynamically registered and inferenced |
| 248 | with concurrently""" |
| 249 | |
| 250 | # This test file configures the server with --max-cpu-loras=2 and this test |
| 251 | # will concurrently load 10 adapters, so it should flex the LRU cache |
| 252 | async def load_and_run_adapter(adapter_name: str): |
| 253 | await client.post( |
| 254 | "load_lora_adapter", |
| 255 | cast_to=str, |
| 256 | body={"lora_name": adapter_name, "lora_path": str(qwen3_lora_files)}, |
| 257 | ) |
| 258 | for _ in range(3): |
| 259 | await client.completions.create( |
| 260 | model=adapter_name, |
| 261 | prompt=["Hello there", "Foo bar bazz buzz"], |
| 262 | max_tokens=5, |
| 263 | ) |
| 264 | |
| 265 | lora_tasks = [] |
| 266 | for i in range(10): |
| 267 | lora_tasks.append(asyncio.create_task(load_and_run_adapter(f"adapter_{i}"))) |
| 268 | |
| 269 | results, _ = await asyncio.wait(lora_tasks) |
| 270 | |
| 271 | for r in results: |
| 272 | assert not isinstance(r, Exception), f"Got exception {r}" |
| 273 | |
| 274 | |
| 275 | @pytest.mark.asyncio |
| 276 | async def test_loading_invalid_adapters_does_not_break_others( |
| 277 | client: openai.AsyncOpenAI, tmp_path, qwen3_lora_files |
| 278 | ): |
| 279 | invalid_files = tmp_path / "invalid_files" |
| 280 | invalid_files.mkdir() |
| 281 | (invalid_files / "adapter_config.json").write_text("this is not json") |
| 282 | |
| 283 | stop_good_requests_event = asyncio.Event() |
| 284 | |
| 285 | async def run_good_requests(client): |
| 286 | # Run chat completions requests until event set |
| 287 | |
| 288 | results = [] |
| 289 | |
| 290 | while not stop_good_requests_event.is_set(): |
| 291 | try: |
| 292 | batch = await client.completions.create( |
| 293 | model="qwen3-lora", |
| 294 | prompt=["Hello there", "Foo bar bazz buzz"], |
| 295 | max_tokens=5, |
| 296 | ) |
| 297 | results.append(batch) |
| 298 | except Exception as e: |
| 299 | results.append(e) |
| 300 | |
| 301 | return results |
| 302 | |
| 303 | # Create task to run good requests |
| 304 | good_task = asyncio.create_task(run_good_requests(client)) |
| 305 | |
| 306 | # Run a bunch of bad adapter loads |
| 307 | for _ in range(25): |
| 308 | with suppress(openai.NotFoundError): |
| 309 | await client.post( |
| 310 | "load_lora_adapter", |
| 311 | cast_to=str, |
| 312 | body={"lora_name": "notfound", "lora_path": "/not/an/adapter"}, |
| 313 | ) |
| 314 | for _ in range(25): |
| 315 | with suppress(openai.InternalServerError): |
| 316 | await client.post( |
| 317 | "load_lora_adapter", |
| 318 | cast_to=str, |
| 319 | body={"lora_name": "invalid", "lora_path": str(invalid_files)}, |
| 320 | ) |
| 321 | |
| 322 | # Ensure all the running requests with lora adapters succeeded |
| 323 | stop_good_requests_event.set() |
| 324 | results = await good_task |
| 325 | for r in results: |
| 326 | assert not isinstance(r, Exception), f"Got exception {r}" |
| 327 | |
| 328 | # Ensure we can load another adapter and run it |
| 329 | await client.post( |
| 330 | "load_lora_adapter", |
| 331 | cast_to=str, |
| 332 | body={"lora_name": "valid", "lora_path": qwen3_lora_files}, |
| 333 | ) |
| 334 | await client.completions.create( |
| 335 | model="valid", |
| 336 | prompt=["Hello there", "Foo bar bazz buzz"], |
| 337 | max_tokens=5, |
| 338 | ) |
| 339 | |
| 340 | |
| 341 | @pytest.mark.asyncio |
| 342 | async def test_beam_search_with_lora_adapters( |
| 343 | client: openai.AsyncOpenAI, |
| 344 | tmp_path, |
| 345 | qwen3_lora_files, |
| 346 | ): |
| 347 | """Validate that async beam search can be used with lora.""" |
| 348 | |
| 349 | async def load_and_run_adapter(adapter_name: str): |
| 350 | await client.post( |
| 351 | "load_lora_adapter", |
| 352 | cast_to=str, |
| 353 | body={"lora_name": adapter_name, "lora_path": str(qwen3_lora_files)}, |
| 354 | ) |
| 355 | for _ in range(3): |
| 356 | await client.completions.create( |
| 357 | model=adapter_name, |
| 358 | prompt=["Hello there", "Foo bar bazz buzz"], |
| 359 | max_tokens=5, |
| 360 | extra_body=dict(use_beam_search=True), |
| 361 | ) |
| 362 | |
| 363 | lora_tasks = [] |
| 364 | for i in range(3): |
| 365 | lora_tasks.append(asyncio.create_task(load_and_run_adapter(f"adapter_{i}"))) |
| 366 | |
| 367 | results, _ = await asyncio.wait(lora_tasks) |
| 368 | |
| 369 | for r in results: |
| 370 | assert not isinstance(r, Exception), f"Got exception {r}" |