A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 23:00 UTC
[Feat] allow inplace loading lora (#31326)
Signed-off-by: Jackmin801 <ongjackm@gmail.com> Signed-off-by: Jackmin801 <56836461+Jackmin801@users.noreply.github.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
Jackmin801 committed
Jan 19, 2026 at 18:15 UTC
12dab78f498f873f562384c4e68abcb192df9566
10 files changed
+262
-7
docs/features/lora.md
+18
@@ -210,6 +210,24 @@ Alternatively, follow these example steps to implement your own plugin:
210
211
For more details, refer to the [vLLM's Plugins System](../design/plugin_system.md).
212
213
+### In-Place LoRA Reloading
214
+
215
+When dynamically loading LoRA adapters, you may need to replace an existing adapter with updated weights while keeping the same name. The `load_inplace` parameter enables this functionality. This commonly occurs in asynchronous reinforcement learning setups, where adapters are continuously updated and swapped in without interrupting ongoing inference.
216
+
217
+When `load_inplace=True`, vLLM will replace the existing adapter with the new one.
218
+
219
+Example request to load or replace a LoRA adapter with the same name:
220
+
221
+```bash
222
+curl -X POST http://localhost:8000/v1/load_lora_adapter \
223
+-H "Content-Type: application/json" \
224
+-d '{
225
+ "lora_name": "my-adapter",
226
+ "lora_path": "/path/to/adapter/v2",
227
+ "load_inplace": true
228
+}'
229
+```
230
+
231
## New format for `--lora-modules`
232
233
In the previous version, users would provide LoRA modules via the following format, either as a key-value pair or in JSON format. For example:
tests/entrypoints/conftest.py
+16
@@ -195,6 +195,22 @@ def qwen3_lora_files():
195
return snapshot_download(repo_id="charent/self_cognition_Alice")
196
197
198
+@pytest.fixture(scope="session")
199
+def qwen3_meowing_lora_files():
200
+ """Download Qwen3 LoRA files once per test session."""
201
+ from huggingface_hub import snapshot_download
202
+
203
+ return snapshot_download(repo_id="Jackmin108/Qwen3-0.6B-Meow-LoRA")
204
+
205
+
206
+@pytest.fixture(scope="session")
207
+def qwen3_woofing_lora_files():
208
+ """Download Qwen3 LoRA files once per test session."""
209
+ from huggingface_hub import snapshot_download
210
+
211
+ return snapshot_download(repo_id="Jackmin108/Qwen3-0.6B-Woof-LoRA")
212
+
213
+
214
@pytest.fixture(scope="session")
215
def opt125_lora_files() -> str:
216
"""Download opt-125m LoRA files once per test session."""
tests/entrypoints/openai/test_lora_adapters.py
+76
@@ -104,6 +104,82 @@ async def test_dynamic_lora_lineage(client: openai.AsyncOpenAI, qwen3_lora_files
104
assert dynamic_lora_model.id == "qwen3-lora-3"
105
106
107
+@pytest.mark.asyncio
108
+async def test_load_lora_adapter_with_same_name_replaces_inplace(
109
+ client: openai.AsyncOpenAI, qwen3_meowing_lora_files, qwen3_woofing_lora_files
110
+):
111
+ """Test that loading a LoRA adapter with the same name replaces it inplace."""
112
+ adapter_name = "replaceable-adapter"
113
+ messages = [
114
+ {"content": "Follow the instructions to make animal noises", "role": "system"},
115
+ {"content": "Make your favorite animal noise.", "role": "user"},
116
+ ]
117
+
118
+ # Load LoRA that makes model meow
119
+ response = await client.post(
120
+ "load_lora_adapter",
121
+ cast_to=str,
122
+ body={"lora_name": adapter_name, "lora_path": qwen3_meowing_lora_files},
123
+ )
124
+ assert "success" in response.lower()
125
+
126
+ completion = await client.chat.completions.create(
127
+ model=adapter_name,
128
+ messages=messages,
129
+ max_tokens=10,
130
+ )
131
+ assert "Meow Meow Meow" in completion.choices[0].message.content
132
+
133
+ # Load LoRA that makes model woof
134
+ response = await client.post(
135
+ "load_lora_adapter",
136
+ cast_to=str,
137
+ body={
138
+ "lora_name": adapter_name,
139
+ "lora_path": qwen3_woofing_lora_files,
140
+ "load_inplace": True,
141
+ },
142
+ )
143
+ assert "success" in response.lower()
144
+
145
+ completion = await client.chat.completions.create(
146
+ model=adapter_name,
147
+ messages=messages,
148
+ max_tokens=10,
149
+ )
150
+ assert "Woof Woof Woof" in completion.choices[0].message.content
151
+
152
+
153
+@pytest.mark.asyncio
154
+async def test_load_lora_adapter_with_load_inplace_false_errors(
155
+ client: openai.AsyncOpenAI, qwen3_meowing_lora_files
156
+):
157
+ """Test that load_inplace=False returns an error when adapter already exists."""
158
+ adapter_name = "test-load-inplace-false"
159
+
160
+ # Load LoRA adapter first time (should succeed)
161
+ response = await client.post(
162
+ "load_lora_adapter",
163
+ cast_to=str,
164
+ body={"lora_name": adapter_name, "lora_path": qwen3_meowing_lora_files},
165
+ )
166
+ assert "success" in response.lower()
167
+
168
+ # Try to load the same adapter again with load_inplace=False (should fail)
169
+ with pytest.raises(openai.BadRequestError) as exc_info:
170
+ await client.post(
171
+ "load_lora_adapter",
172
+ cast_to=str,
173
+ body={
174
+ "lora_name": adapter_name,
175
+ "lora_path": qwen3_meowing_lora_files,
176
+ },
177
+ )
178
+
179
+ # Verify the error message
180
+ assert "already been loaded" in str(exc_info.value)
181
+
182
+
183
@pytest.mark.asyncio
184
async def test_dynamic_lora_not_found(client: openai.AsyncOpenAI):
185
with pytest.raises(openai.NotFoundError):
tests/lora/conftest.py
+12
@@ -233,6 +233,18 @@ def qwen3vl_vision_lora_files():
233
return snapshot_download(repo_id="EpochEcho/qwen3-4b-vl-lora-vision-connector")
234
235
236
+@pytest.fixture(scope="session")
237
+def qwen3_meowing_lora_files():
238
+ """Download Qwen3 Meow LoRA files once per test session."""
239
+ return snapshot_download(repo_id="Jackmin108/Qwen3-0.6B-Meow-LoRA")
240
+
241
+
242
+@pytest.fixture(scope="session")
243
+def qwen3_woofing_lora_files():
244
+ """Download Qwen3 Woof LoRA files once per test session."""
245
+ return snapshot_download(repo_id="Jackmin108/Qwen3-0.6B-Woof-LoRA")
246
+
247
+
248
@pytest.fixture(scope="session")
249
def tinyllama_lora_files():
250
return snapshot_download(repo_id="jashing/tinyllama-colorist-lora")
tests/lora/test_llm_with_multi_loras.py
+111
-2
@@ -30,9 +30,11 @@ LORA_TEST_EXPECTED = [
30
]
31
32
33
-def format_chatml_messages(prompt: str):
33
+def format_chatml_messages(
34
+ prompt: str, system_prompt: str = "You are a helpful assistant."
35
+) -> list[dict[str, str]]:
36
return [
35
- {"role": "system", "content": "You are a helpful assistant."},
37
+ {"role": "system", "content": system_prompt},
38
{"role": "user", "content": prompt},
39
]
40
@@ -185,3 +187,110 @@ def test_multiple_lora_requests():
187
single_lora_request = lora_request[0]
188
outputs = llm.generate(PROMPTS, lora_request=single_lora_request)
189
assert len(PROMPTS) == len(outputs)
190
+
191
+
192
+def test_load_inplace_offline_reload(
193
+ qwen3_meowing_lora_files: str, qwen3_woofing_lora_files: str
194
+) -> None:
195
+ """
196
+ Test that load_inplace=True allows reloading LoRA adapters with the same ID
197
+ in offline mode (using LLM class directly).
198
+ """
199
+ llm = LLM(
200
+ model=MODEL_PATH,
201
+ enable_lora=True,
202
+ max_loras=2,
203
+ max_lora_rank=LORA_RANK,
204
+ max_model_len=512,
205
+ gpu_memory_utilization=0.5,
206
+ enforce_eager=True,
207
+ )
208
+ adapter_id = 1
209
+ messages = format_chatml_messages(
210
+ "Make your favorite animal noise.",
211
+ system_prompt="Follow the instructions to make animal noises",
212
+ )
213
+ sampling_params = SamplingParams(temperature=0, max_tokens=10)
214
+
215
+ # Load meowing LoRA with load_inplace=True
216
+ meowing_request = LoRARequest(
217
+ lora_name="test-adapter",
218
+ lora_int_id=adapter_id,
219
+ lora_path=qwen3_meowing_lora_files,
220
+ )
221
+
222
+ outputs = llm.chat([messages], sampling_params, lora_request=meowing_request)
223
+ first_output = outputs[0].outputs[0].text.strip()
224
+ assert "Meow Meow Meow" in first_output, (
225
+ f"Expected meowing output, got: {first_output}"
226
+ )
227
+
228
+ # Reload with woofing LoRA (same ID, different weights, load_inplace=True)
229
+ woofing_request = LoRARequest(
230
+ lora_name="test-adapter-woof",
231
+ lora_int_id=adapter_id, # Same ID
232
+ lora_path=qwen3_woofing_lora_files, # Different weights
233
+ load_inplace=True, # Force reload
234
+ )
235
+
236
+ outputs = llm.chat([messages], sampling_params, lora_request=woofing_request)
237
+ second_output = outputs[0].outputs[0].text.strip()
238
+ assert "Woof Woof Woof" in second_output, (
239
+ f"Expected woofing output, got: {second_output}"
240
+ )
241
+
242
+
243
+def test_load_inplace_false_no_reload(
244
+ qwen3_meowing_lora_files: str, qwen3_woofing_lora_files: str
245
+) -> None:
246
+ """
247
+ Test that load_inplace=False prevents reloading when an adapter
248
+ with the same ID already exists.
249
+ """
250
+ llm = LLM(
251
+ model=MODEL_PATH,
252
+ enable_lora=True,
253
+ max_loras=2,
254
+ max_lora_rank=LORA_RANK,
255
+ max_model_len=512,
256
+ gpu_memory_utilization=0.5,
257
+ enforce_eager=True,
258
+ )
259
+ adapter_id = 2
260
+ messages = format_chatml_messages(
261
+ "Make your favorite animal noise.",
262
+ system_prompt="Follow the instructions to make animal noises",
263
+ )
264
+ sampling_params = SamplingParams(temperature=0, max_tokens=10)
265
+
266
+ # Load meowing LoRA first with load_inplace=True
267
+ meowing_request_initial = LoRARequest(
268
+ lora_name="test-adapter-2",
269
+ lora_int_id=adapter_id,
270
+ lora_path=qwen3_meowing_lora_files,
271
+ )
272
+
273
+ outputs = llm.chat(
274
+ [messages], sampling_params, lora_request=meowing_request_initial
275
+ )
276
+ first_output = outputs[0].outputs[0].text.strip()
277
+ assert "Meow Meow Meow" in first_output, (
278
+ f"Expected meowing output, got: {first_output}"
279
+ )
280
+
281
+ # Try to load woofing LoRA with same ID but load_inplace=False
282
+ # This should NOT reload (adapter 2 already exists)
283
+ woofing_request_no_reload = LoRARequest(
284
+ lora_name="test-adapter-2-woof",
285
+ lora_int_id=adapter_id, # Same ID
286
+ lora_path=qwen3_woofing_lora_files,
287
+ )
288
+
289
+ outputs = llm.chat(
290
+ [messages], sampling_params, lora_request=woofing_request_no_reload
291
+ )
292
+ second_output = outputs[0].outputs[0].text.strip()
293
+ # Should still get meowing output because it didn't reload
294
+ assert "Meow Meow Meow" in second_output, (
295
+ f"Expected meowing output (no reload), got: {second_output}"
296
+ )
vllm/entrypoints/openai/models/serving.py
+13
-4
@@ -132,9 +132,16 @@ class OpenAIServingModels:
132
return error_check_ret
133
134
lora_path = request.lora_path
135
- unique_id = self.lora_id_counter.inc(1)
135
+ lora_int_id = (
136
+ self.lora_requests[lora_name].lora_int_id
137
+ if lora_name in self.lora_requests
138
+ else self.lora_id_counter.inc(1)
139
+ )
140
lora_request = LoRARequest(
137
- lora_name=lora_name, lora_int_id=unique_id, lora_path=lora_path
141
+ lora_name=lora_name,
142
+ lora_int_id=lora_int_id,
143
+ lora_path=lora_path,
144
+ load_inplace=request.load_inplace,
145
)
146
if base_model_name is not None and self.is_base_model(base_model_name):
147
lora_request.base_model_name = base_model_name
@@ -187,11 +194,13 @@ class OpenAIServingModels:
194
status_code=HTTPStatus.BAD_REQUEST,
195
)
196
197
+ # If not loading inplace
198
# Check if the lora adapter with the given name already exists
191
- if request.lora_name in self.lora_requests:
199
+ if not request.load_inplace and request.lora_name in self.lora_requests:
200
return create_error_response(
201
message=f"The lora adapter '{request.lora_name}' has already been "
194
- "loaded.",
202
+ "loaded. If you want to load the adapter in place, set 'load_inplace'"
203
+ " to True.",
204
err_type="InvalidUserInput",
205
status_code=HTTPStatus.BAD_REQUEST,
206
)
vllm/entrypoints/serve/lora/api_router.py
+1
@@ -36,6 +36,7 @@ def attach_router(app: FastAPI):
36
request_shape={
37
"lora_name": "body.name",
38
"lora_path": "body.src",
39
+ "load_inplace": "body.load_inplace || `false`",
40
},
41
)
42
@router.post("/v1/load_lora_adapter", dependencies=[Depends(validate_json_request)])
vllm/entrypoints/serve/lora/protocol.py
+1
@@ -7,6 +7,7 @@ from pydantic import BaseModel, Field
7
class LoadLoRAAdapterRequest(BaseModel):
8
lora_name: str
9
lora_path: str
10
+ load_inplace: bool = False
11
12
13
class UnloadLoRAAdapterRequest(BaseModel):
vllm/lora/request.py
+6
@@ -15,6 +15,11 @@ class LoRARequest(
15
16
lora_int_id must be globally unique for a given adapter.
17
This is currently not enforced in vLLM.
18
+
19
+ load_inplace: If True, forces reloading the adapter even if one
20
+ with the same lora_int_id already exists in the cache. This replaces
21
+ the existing adapter in-place. If False (default), only loads if the
22
+ adapter is not already loaded.
23
"""
24
25
lora_name: str
@@ -22,6 +27,7 @@ class LoRARequest(
27
lora_path: str = ""
28
base_model_name: str | None = msgspec.field(default=None)
29
tensorizer_config_dict: dict | None = None
30
+ load_inplace: bool = False
31
32
def __post_init__(self):
33
if self.lora_int_id < 1:
vllm/lora/worker_manager.py
+8
-1
@@ -254,13 +254,20 @@ class LRUCacheWorkerLoRAManager(WorkerLoRAManager):
254
# This is ok because it's currently only called from
255
# the single-threaded core engine loop.
256
257
- if lora_request.lora_int_id not in self.list_adapters():
257
+ if (
258
+ lora_request.lora_int_id not in self.list_adapters()
259
+ or lora_request.load_inplace
260
+ ):
261
# Load the new adapter first to ensure it is actually valid, before
262
# evicting any existing adapters.
263
# This may cause the # of loaded lora adapters to very temporarily
264
# exceed `--max-cpu-loras`.
265
lora = self._load_adapter(lora_request)
266
267
+ # Remove the existing adapter if it exists
268
+ # Use case for LoRA inplace
269
+ self._adapter_manager.remove_adapter(lora.id)
270
+
271
# Loading succeeded, now check if we will exceed cache capacity and
272
# evict if the oldest adapter if so
273
if len(self._adapter_manager) + 1 > self._adapter_manager.capacity: