A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[Bugfix] Validate lora adapters to avoid crashing server (#11727)
Signed-off-by: Joe Runde <Joseph.Runde@ibm.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
Joe Runde committed
Jan 10, 2025 at 00:56 UTC
ac2f3f7fee93cf9cd97c0078e362feab7b6c8299
15 files changed
+459
-171
tests/entrypoints/openai/test_lora_adapters.py
new
+269
@@ -0,0 +1,269 @@
1
+import asyncio
2
+import json
3
+import shutil
4
+from contextlib import suppress
5
+
6
+import openai # use the official client for correctness check
7
+import pytest
8
+import pytest_asyncio
9
+# downloading lora to test lora requests
10
+from huggingface_hub import snapshot_download
11
+
12
+from ...utils import RemoteOpenAIServer
13
+
14
+# any model with a chat template should work here
15
+MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
16
+# technically this needs Mistral-7B-v0.1 as base, but we're not testing
17
+# generation quality here
18
+LORA_NAME = "typeof/zephyr-7b-beta-lora"
19
+
20
+
21
+@pytest.fixture(scope="module")
22
+def zephyr_lora_files():
23
+ return snapshot_download(repo_id=LORA_NAME)
24
+
25
+
26
+@pytest.fixture(scope="module")
27
+def server_with_lora_modules_json(zephyr_lora_files):
28
+ # Define the json format LoRA module configurations
29
+ lora_module_1 = {
30
+ "name": "zephyr-lora",
31
+ "path": zephyr_lora_files,
32
+ "base_model_name": MODEL_NAME
33
+ }
34
+
35
+ lora_module_2 = {
36
+ "name": "zephyr-lora2",
37
+ "path": zephyr_lora_files,
38
+ "base_model_name": MODEL_NAME
39
+ }
40
+
41
+ args = [
42
+ # use half precision for speed and memory savings in CI environment
43
+ "--dtype",
44
+ "bfloat16",
45
+ "--max-model-len",
46
+ "8192",
47
+ "--enforce-eager",
48
+ # lora config below
49
+ "--enable-lora",
50
+ "--lora-modules",
51
+ json.dumps(lora_module_1),
52
+ json.dumps(lora_module_2),
53
+ "--max-lora-rank",
54
+ "64",
55
+ "--max-cpu-loras",
56
+ "2",
57
+ "--max-num-seqs",
58
+ "64",
59
+ ]
60
+
61
+ # Enable the /v1/load_lora_adapter endpoint
62
+ envs = {"VLLM_ALLOW_RUNTIME_LORA_UPDATING": "True"}
63
+
64
+ with RemoteOpenAIServer(MODEL_NAME, args, env_dict=envs) as remote_server:
65
+ yield remote_server
66
+
67
+
68
+@pytest_asyncio.fixture
69
+async def client(server_with_lora_modules_json):
70
+ async with server_with_lora_modules_json.get_async_client(
71
+ ) as async_client:
72
+ yield async_client
73
+
74
+
75
+@pytest.mark.asyncio
76
+async def test_static_lora_lineage(client: openai.AsyncOpenAI,
77
+ zephyr_lora_files):
78
+ models = await client.models.list()
79
+ models = models.data
80
+ served_model = models[0]
81
+ lora_models = models[1:]
82
+ assert served_model.id == MODEL_NAME
83
+ assert served_model.root == MODEL_NAME
84
+ assert served_model.parent is None
85
+ assert all(lora_model.root == zephyr_lora_files
86
+ for lora_model in lora_models)
87
+ assert all(lora_model.parent == MODEL_NAME for lora_model in lora_models)
88
+ assert lora_models[0].id == "zephyr-lora"
89
+ assert lora_models[1].id == "zephyr-lora2"
90
+
91
+
92
+@pytest.mark.asyncio
93
+async def test_dynamic_lora_lineage(client: openai.AsyncOpenAI,
94
+ zephyr_lora_files):
95
+
96
+ response = await client.post("load_lora_adapter",
97
+ cast_to=str,
98
+ body={
99
+ "lora_name": "zephyr-lora-3",
100
+ "lora_path": zephyr_lora_files
101
+ })
102
+ # Ensure adapter loads before querying /models
103
+ assert "success" in response
104
+
105
+ models = await client.models.list()
106
+ models = models.data
107
+ dynamic_lora_model = models[-1]
108
+ assert dynamic_lora_model.root == zephyr_lora_files
109
+ assert dynamic_lora_model.parent == MODEL_NAME
110
+ assert dynamic_lora_model.id == "zephyr-lora-3"
111
+
112
+
113
+@pytest.mark.asyncio
114
+async def test_dynamic_lora_not_found(client: openai.AsyncOpenAI):
115
+ with pytest.raises(openai.NotFoundError):
116
+ await client.post("load_lora_adapter",
117
+ cast_to=str,
118
+ body={
119
+ "lora_name": "notfound",
120
+ "lora_path": "/not/an/adapter"
121
+ })
122
+
123
+
124
+@pytest.mark.asyncio
125
+async def test_dynamic_lora_invalid_files(client: openai.AsyncOpenAI,
126
+ tmp_path):
127
+ invalid_files = tmp_path / "invalid_files"
128
+ invalid_files.mkdir()
129
+ (invalid_files / "adapter_config.json").write_text("this is not json")
130
+
131
+ with pytest.raises(openai.BadRequestError):
132
+ await client.post("load_lora_adapter",
133
+ cast_to=str,
134
+ body={
135
+ "lora_name": "invalid-json",
136
+ "lora_path": str(invalid_files)
137
+ })
138
+
139
+
140
+@pytest.mark.asyncio
141
+async def test_dynamic_lora_invalid_lora_rank(client: openai.AsyncOpenAI,
142
+ tmp_path, zephyr_lora_files):
143
+ invalid_rank = tmp_path / "invalid_rank"
144
+
145
+ # Copy adapter from zephyr_lora_files to invalid_rank
146
+ shutil.copytree(zephyr_lora_files, invalid_rank)
147
+
148
+ with open(invalid_rank / "adapter_config.json") as f:
149
+ adapter_config = json.load(f)
150
+
151
+ print(adapter_config)
152
+
153
+ # assert False
154
+
155
+ # Change rank to invalid value
156
+ adapter_config["r"] = 1024
157
+ with open(invalid_rank / "adapter_config.json", "w") as f:
158
+ json.dump(adapter_config, f)
159
+
160
+ with pytest.raises(openai.BadRequestError,
161
+ match="is greater than max_lora_rank"):
162
+ await client.post("load_lora_adapter",
163
+ cast_to=str,
164
+ body={
165
+ "lora_name": "invalid-json",
166
+ "lora_path": str(invalid_rank)
167
+ })
168
+
169
+
170
+@pytest.mark.asyncio
171
+async def test_multiple_lora_adapters(client: openai.AsyncOpenAI, tmp_path,
172
+ zephyr_lora_files):
173
+ """Validate that many loras can be dynamically registered and inferenced
174
+ with concurrently"""
175
+
176
+ # This test file configures the server with --max-cpu-loras=2 and this test
177
+ # will concurrently load 10 adapters, so it should flex the LRU cache
178
+ async def load_and_run_adapter(adapter_name: str):
179
+ await client.post("load_lora_adapter",
180
+ cast_to=str,
181
+ body={
182
+ "lora_name": adapter_name,
183
+ "lora_path": str(zephyr_lora_files)
184
+ })
185
+ for _ in range(3):
186
+ await client.completions.create(
187
+ model=adapter_name,
188
+ prompt=["Hello there", "Foo bar bazz buzz"],
189
+ max_tokens=5,
190
+ )
191
+
192
+ lora_tasks = []
193
+ for i in range(10):
194
+ lora_tasks.append(
195
+ asyncio.create_task(load_and_run_adapter(f"adapter_{i}")))
196
+
197
+ results, _ = await asyncio.wait(lora_tasks)
198
+
199
+ for r in results:
200
+ assert not isinstance(r, Exception), f"Got exception {r}"
201
+
202
+
203
+@pytest.mark.asyncio
204
+async def test_loading_invalid_adapters_does_not_break_others(
205
+ client: openai.AsyncOpenAI, tmp_path, zephyr_lora_files):
206
+
207
+ invalid_files = tmp_path / "invalid_files"
208
+ invalid_files.mkdir()
209
+ (invalid_files / "adapter_config.json").write_text("this is not json")
210
+
211
+ stop_good_requests_event = asyncio.Event()
212
+
213
+ async def run_good_requests(client):
214
+ # Run chat completions requests until event set
215
+
216
+ results = []
217
+
218
+ while not stop_good_requests_event.is_set():
219
+ try:
220
+ batch = await client.completions.create(
221
+ model="zephyr-lora",
222
+ prompt=["Hello there", "Foo bar bazz buzz"],
223
+ max_tokens=5,
224
+ )
225
+ results.append(batch)
226
+ except Exception as e:
227
+ results.append(e)
228
+
229
+ return results
230
+
231
+ # Create task to run good requests
232
+ good_task = asyncio.create_task(run_good_requests(client))
233
+
234
+ # Run a bunch of bad adapter loads
235
+ for _ in range(25):
236
+ with suppress(openai.NotFoundError):
237
+ await client.post("load_lora_adapter",
238
+ cast_to=str,
239
+ body={
240
+ "lora_name": "notfound",
241
+ "lora_path": "/not/an/adapter"
242
+ })
243
+ for _ in range(25):
244
+ with suppress(openai.BadRequestError):
245
+ await client.post("load_lora_adapter",
246
+ cast_to=str,
247
+ body={
248
+ "lora_name": "invalid",
249
+ "lora_path": str(invalid_files)
250
+ })
251
+
252
+ # Ensure all the running requests with lora adapters succeeded
253
+ stop_good_requests_event.set()
254
+ results = await good_task
255
+ for r in results:
256
+ assert not isinstance(r, Exception), f"Got exception {r}"
257
+
258
+ # Ensure we can load another adapter and run it
259
+ await client.post("load_lora_adapter",
260
+ cast_to=str,
261
+ body={
262
+ "lora_name": "valid",
263
+ "lora_path": zephyr_lora_files
264
+ })
265
+ await client.completions.create(
266
+ model="valid",
267
+ prompt=["Hello there", "Foo bar bazz buzz"],
268
+ max_tokens=5,
269
+ )
tests/entrypoints/openai/test_lora_lineage.py
deleted
-109
@@ -1,109 +0,0 @@
1
-import json
2
-
3
-import openai # use the official client for correctness check
4
-import pytest
5
-import pytest_asyncio
6
-# downloading lora to test lora requests
7
-from huggingface_hub import snapshot_download
8
-
9
-from ...utils import RemoteOpenAIServer
10
-
11
-# any model with a chat template should work here
12
-MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
13
-# technically this needs Mistral-7B-v0.1 as base, but we're not testing
14
-# generation quality here
15
-LORA_NAME = "typeof/zephyr-7b-beta-lora"
16
-
17
-
18
-@pytest.fixture(scope="module")
19
-def zephyr_lora_files():
20
- return snapshot_download(repo_id=LORA_NAME)
21
-
22
-
23
-@pytest.fixture(scope="module")
24
-def server_with_lora_modules_json(zephyr_lora_files):
25
- # Define the json format LoRA module configurations
26
- lora_module_1 = {
27
- "name": "zephyr-lora",
28
- "path": zephyr_lora_files,
29
- "base_model_name": MODEL_NAME
30
- }
31
-
32
- lora_module_2 = {
33
- "name": "zephyr-lora2",
34
- "path": zephyr_lora_files,
35
- "base_model_name": MODEL_NAME
36
- }
37
-
38
- args = [
39
- # use half precision for speed and memory savings in CI environment
40
- "--dtype",
41
- "bfloat16",
42
- "--max-model-len",
43
- "8192",
44
- "--enforce-eager",
45
- # lora config below
46
- "--enable-lora",
47
- "--lora-modules",
48
- json.dumps(lora_module_1),
49
- json.dumps(lora_module_2),
50
- "--max-lora-rank",
51
- "64",
52
- "--max-cpu-loras",
53
- "2",
54
- "--max-num-seqs",
55
- "64",
56
- ]
57
-
58
- # Enable the /v1/load_lora_adapter endpoint
59
- envs = {"VLLM_ALLOW_RUNTIME_LORA_UPDATING": "True"}
60
-
61
- with RemoteOpenAIServer(MODEL_NAME, args, env_dict=envs) as remote_server:
62
- yield remote_server
63
-
64
-
65
-@pytest_asyncio.fixture
66
-async def client_for_lora_lineage(server_with_lora_modules_json):
67
- async with server_with_lora_modules_json.get_async_client(
68
- ) as async_client:
69
- yield async_client
70
-
71
-
72
-@pytest.mark.asyncio
73
-async def test_static_lora_lineage(client_for_lora_lineage: openai.AsyncOpenAI,
74
- zephyr_lora_files):
75
- models = await client_for_lora_lineage.models.list()
76
- models = models.data
77
- served_model = models[0]
78
- lora_models = models[1:]
79
- assert served_model.id == MODEL_NAME
80
- assert served_model.root == MODEL_NAME
81
- assert served_model.parent is None
82
- assert all(lora_model.root == zephyr_lora_files
83
- for lora_model in lora_models)
84
- assert all(lora_model.parent == MODEL_NAME for lora_model in lora_models)
85
- assert lora_models[0].id == "zephyr-lora"
86
- assert lora_models[1].id == "zephyr-lora2"
87
-
88
-
89
-@pytest.mark.asyncio
90
-async def test_dynamic_lora_lineage(
91
- client_for_lora_lineage: openai.AsyncOpenAI, zephyr_lora_files):
92
-
93
- response = await client_for_lora_lineage.post("load_lora_adapter",
94
- cast_to=str,
95
- body={
96
- "lora_name":
97
- "zephyr-lora-3",
98
- "lora_path":
99
- zephyr_lora_files
100
- })
101
- # Ensure adapter loads before querying /models
102
- assert "success" in response
103
-
104
- models = await client_for_lora_lineage.models.list()
105
- models = models.data
106
- dynamic_lora_model = models[-1]
107
- assert dynamic_lora_model.root == zephyr_lora_files
108
- assert dynamic_lora_model.parent == MODEL_NAME
109
- assert dynamic_lora_model.id == "zephyr-lora-3"
tests/entrypoints/openai/test_serving_chat.py
+5
-3
@@ -52,7 +52,7 @@ async def _async_serving_chat_init():
52
engine = MockEngine()
53
model_config = await engine.get_model_config()
54
55
- models = OpenAIServingModels(model_config, BASE_MODEL_PATHS)
55
+ models = OpenAIServingModels(engine, model_config, BASE_MODEL_PATHS)
56
serving_completion = OpenAIServingChat(engine,
57
model_config,
58
models,
@@ -73,7 +73,8 @@ def test_serving_chat_should_set_correct_max_tokens():
73
mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME)
74
mock_engine.errored = False
75
76
- models = OpenAIServingModels(base_model_paths=BASE_MODEL_PATHS,
76
+ models = OpenAIServingModels(engine_client=mock_engine,
77
+ base_model_paths=BASE_MODEL_PATHS,
78
model_config=MockModelConfig())
79
serving_chat = OpenAIServingChat(mock_engine,
80
MockModelConfig(),
@@ -116,7 +117,8 @@ def test_serving_chat_could_load_correct_generation_config():
117
mock_engine.errored = False
118
119
# Initialize the serving chat
119
- models = OpenAIServingModels(base_model_paths=BASE_MODEL_PATHS,
120
+ models = OpenAIServingModels(engine_client=mock_engine,
121
+ base_model_paths=BASE_MODEL_PATHS,
122
model_config=mock_model_config)
123
serving_chat = OpenAIServingChat(mock_engine,
124
mock_model_config,
tests/entrypoints/openai/test_serving_models.py
+7
-3
@@ -4,6 +4,7 @@ from unittest.mock import MagicMock
4
import pytest
5
6
from vllm.config import ModelConfig
7
+from vllm.engine.protocol import EngineClient
8
from vllm.entrypoints.openai.protocol import (ErrorResponse,
9
LoadLoraAdapterRequest,
10
UnloadLoraAdapterRequest)
@@ -21,13 +22,16 @@ LORA_UNLOADING_SUCCESS_MESSAGE = (
22
23
async def _async_serving_models_init() -> OpenAIServingModels:
24
mock_model_config = MagicMock(spec=ModelConfig)
25
+ mock_engine_client = MagicMock(spec=EngineClient)
26
# Set the max_model_len attribute to avoid missing attribute
27
mock_model_config.max_model_len = 2048
28
27
- serving_models = OpenAIServingModels(base_model_paths=BASE_MODEL_PATHS,
29
+ serving_models = OpenAIServingModels(engine_client=mock_engine_client,
30
+ base_model_paths=BASE_MODEL_PATHS,
31
model_config=mock_model_config,
32
lora_modules=None,
33
prompt_adapters=None)
34
+ await serving_models.init_static_loras()
35
36
return serving_models
37
@@ -113,5 +117,5 @@ async def test_unload_lora_adapter_not_found():
117
request = UnloadLoraAdapterRequest(lora_name="nonexistent_adapter")
118
response = await serving_models.unload_lora_adapter(request)
119
assert isinstance(response, ErrorResponse)
116
- assert response.type == "InvalidUserInput"
117
- assert response.code == HTTPStatus.BAD_REQUEST
120
+ assert response.type == "NotFoundError"
121
+ assert response.code == HTTPStatus.NOT_FOUND
tests/entrypoints/openai/test_shutdown.py
+8
-19
@@ -1,6 +1,3 @@
1
-import json
2
-import os
3
-
1
import openai
2
import pytest
3
@@ -10,16 +7,7 @@ MODEL_NAME = "meta-llama/Llama-3.2-1B"
7
8
9
@pytest.mark.asyncio
13
-async def test_shutdown_on_engine_failure(tmp_path):
14
- # Use a bad adapter to crash the engine
15
- # (This test will fail when that bug is fixed)
16
- adapter_path = tmp_path / "bad_adapter"
17
- os.mkdir(adapter_path)
18
- with open(adapter_path / "adapter_model_config.json", "w") as f:
19
- json.dump({"not": "real"}, f)
20
- with open(adapter_path / "adapter_model.safetensors", "wb") as f:
21
- f.write(b"this is fake")
22
-
10
+async def test_shutdown_on_engine_failure():
11
# dtype, max-len etc set so that this can run in CI
12
args = [
13
"--dtype",
@@ -29,9 +17,6 @@ async def test_shutdown_on_engine_failure(tmp_path):
17
"--enforce-eager",
18
"--max-num-seqs",
19
"128",
32
- "--enable-lora",
33
- "--lora-modules",
34
- f"bad-adapter={tmp_path / 'bad_adapter'}",
20
]
21
22
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
@@ -39,9 +24,13 @@ async def test_shutdown_on_engine_failure(tmp_path):
24
25
with pytest.raises(
26
(openai.APIConnectionError, openai.InternalServerError)):
42
- # This crashes the engine
43
- await client.completions.create(model="bad-adapter",
44
- prompt="Hello, my name is")
27
+ # Asking for lots of prompt logprobs will currently crash the
28
+ # engine. This may change in the future when that bug is fixed
29
+ prompt = "Hello " * 4000
30
+ await client.completions.create(
31
+ model=MODEL_NAME,
32
+ prompt=prompt,
33
+ extra_body={"prompt_logprobs": 10})
34
35
# Now the server should shut down
36
return_code = remote_server.proc.wait(timeout=8)
vllm/engine/async_llm_engine.py
+4
@@ -1257,6 +1257,10 @@ class AsyncLLMEngine(EngineClient):
1257
else:
1258
self.engine.model_executor._run_workers("stop_profile")
1259
1260
+ async def add_lora(self, lora_request: LoRARequest) -> None:
1261
+ """Load a new LoRA adapter into the engine for future requests."""
1262
+ self.engine.add_lora(lora_request)
1263
+
1264
1265
# TODO(v1): Remove this class proxy when V1 goes default.
1266
if envs.VLLM_USE_V1:
vllm/engine/multiprocessing/__init__.py
+17
-3
@@ -1,4 +1,5 @@
1
-from dataclasses import dataclass
1
+import uuid
2
+from dataclasses import dataclass, field
3
from enum import Enum
4
from typing import List, Mapping, Optional, Union, overload
5
@@ -120,10 +121,23 @@ class RPCUProfileRequest(Enum):
121
STOP_PROFILE = 2
122
123
124
+@dataclass
125
+class RPCLoadAdapterRequest:
126
+ lora_request: LoRARequest
127
+ # Set the default value of request_id to a new UUID
128
+ request_id: str = field(default_factory=lambda: str(uuid.uuid4()))
129
+
130
+
131
+@dataclass
132
+class RPCAdapterLoadedResponse:
133
+ request_id: str
134
+
135
+
136
RPC_REQUEST_T = Union[RPCProcessRequest, RPCAbortRequest, RPCStartupRequest,
124
- RPCUProfileRequest]
137
+ RPCUProfileRequest, RPCLoadAdapterRequest]
138
126
-REQUEST_OUTPUTS_T = Union[List[RequestOutput], RPCError]
139
+REQUEST_OUTPUTS_T = Union[List[RequestOutput], RPCAdapterLoadedResponse,
140
+ RPCError]
141
142
143
def ENGINE_DEAD_ERROR(
vllm/engine/multiprocessing/client.py
+35
-7
@@ -25,8 +25,10 @@ from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT,
25
IPC_HEALTH_EXT, IPC_INPUT_EXT,
26
IPC_OUTPUT_EXT, RPC_REQUEST_T,
27
VLLM_RPC_SUCCESS_STR, RPCAbortRequest,
28
- RPCError, RPCProcessRequest,
29
- RPCStartupRequest, RPCStartupResponse,
28
+ RPCAdapterLoadedResponse, RPCError,
29
+ RPCLoadAdapterRequest,
30
+ RPCProcessRequest, RPCStartupRequest,
31
+ RPCStartupResponse,
32
RPCUProfileRequest)
33
from vllm.engine.protocol import EngineClient
34
# yapf: enable
@@ -240,17 +242,22 @@ class MQLLMEngineClient(EngineClient):
242
queue = self.output_queues.get(request_id)
243
if queue is not None:
244
queue.put_nowait(exception)
245
+ # Put each output into the appropriate queue.
246
+ elif isinstance(request_outputs, RPCAdapterLoadedResponse):
247
+ self._add_output(request_outputs)
248
else:
244
- # Put each output into the appropriate steam.
249
for request_output in request_outputs:
246
- queue = self.output_queues.get(
247
- request_output.request_id)
248
- if queue is not None:
249
- queue.put_nowait(request_output)
250
+ self._add_output(request_output)
251
252
except asyncio.CancelledError:
253
logger.debug("Shutting down MQLLMEngineClient output handler.")
254
255
+ def _add_output(self, request_output: Union[RequestOutput,
256
+ RPCAdapterLoadedResponse]):
257
+ queue = self.output_queues.get(request_output.request_id)
258
+ if queue is not None:
259
+ queue.put_nowait(request_output)
260
+
261
async def setup(self):
262
"""Setup the client before it starts sending server requests."""
263
@@ -659,3 +666,24 @@ class MQLLMEngineClient(EngineClient):
666
667
await self._send_one_way_rpc_request(
668
request=RPCUProfileRequest.STOP_PROFILE, socket=self.input_socket)
669
+
670
+ async def add_lora(self, lora_request: LoRARequest) -> None:
671
+ """Load a new LoRA adapter into the engine for future requests."""
672
+ # Uses the same I/O as generate requests
673
+ request = RPCLoadAdapterRequest(lora_request)
674
+
675
+ # Create output queue for this requests.
676
+ queue: asyncio.Queue[Union[None, BaseException]] = asyncio.Queue()
677
+ self.output_queues[request.request_id] = queue
678
+
679
+ # Send the request
680
+ request_bytes = pickle.dumps(request)
681
+ await self.input_socket.send_multipart((request_bytes, ), copy=False)
682
+
683
+ # Wait for the response
684
+ request_output = await queue.get()
685
+ self.output_queues.pop(request.request_id)
686
+
687
+ # Raise on error, otherwise happily return None
688
+ if isinstance(request_output, BaseException):
689
+ raise request_output
vllm/engine/multiprocessing/engine.py
+24
-3
@@ -14,8 +14,10 @@ from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT,
14
IPC_HEALTH_EXT, IPC_INPUT_EXT,
15
IPC_OUTPUT_EXT, REQUEST_OUTPUTS_T,
16
VLLM_RPC_SUCCESS_STR, RPCAbortRequest,
17
- RPCError, RPCProcessRequest,
18
- RPCStartupRequest, RPCStartupResponse,
17
+ RPCAdapterLoadedResponse, RPCError,
18
+ RPCLoadAdapterRequest,
19
+ RPCProcessRequest, RPCStartupRequest,
20
+ RPCStartupResponse,
21
RPCUProfileRequest)
22
# yapf: enable
23
from vllm.executor.gpu_executor import GPUExecutor
@@ -234,6 +236,8 @@ class MQLLMEngine:
236
self.start_profile()
237
else:
238
self.stop_profile()
239
+ elif isinstance(request, RPCLoadAdapterRequest):
240
+ self._handle_load_adapter_request(request)
241
else:
242
raise ValueError("Unknown RPCRequest Type: "
243
f"{type(request)}")
@@ -284,6 +288,19 @@ class MQLLMEngine:
288
if self.log_requests:
289
logger.info("Aborted request %s.", request.request_id)
290
291
+ def _handle_load_adapter_request(self, request: RPCLoadAdapterRequest):
292
+ try:
293
+ self.engine.add_lora(request.lora_request)
294
+ except BaseException as e:
295
+ # Send back an error if the adater fails to load
296
+ rpc_err = RPCError(request_id=request.request_id,
297
+ is_engine_errored=False,
298
+ exception=e)
299
+ self._send_outputs(rpc_err)
300
+ # Otherwise, send back the successful load message
301
+ self._send_outputs(
302
+ RPCAdapterLoadedResponse(request_id=request.request_id))
303
+
304
def _health_check(self):
305
# Send unhealthy if engine has already errored
306
if self._errored_with is not None:
@@ -296,7 +313,11 @@ class MQLLMEngine:
313
self._send_unhealthy(e)
314
315
def _send_outputs(self, outputs: REQUEST_OUTPUTS_T):
299
- """Send List of RequestOutput to RPCClient."""
316
+ """Send outputs back to the engine client. These can be:
317
+ - Exceptions
318
+ - A list of generation outputs
319
+ - A response from loading a lora adapter
320
+ """
321
if outputs:
322
try:
323
from ray.exceptions import RayTaskError
vllm/engine/protocol.py
+5
@@ -270,3 +270,8 @@ class EngineClient(ABC):
270
async def stop_profile(self) -> None:
271
"""Start profiling the engine"""
272
...
273
+
274
+ @abstractmethod
275
+ async def add_lora(self, lora_request: LoRARequest) -> None:
276
+ """Load a new LoRA adapter into the engine for future requests."""
277
+ ...
vllm/entrypoints/openai/api_server.py
+4
-3
@@ -662,7 +662,7 @@ def build_app(args: Namespace) -> FastAPI:
662
return app
663
664
665
-def init_app_state(
665
+async def init_app_state(
666
engine_client: EngineClient,
667
model_config: ModelConfig,
668
state: State,
@@ -690,12 +690,13 @@ def init_app_state(
690
logger.info("Using supplied chat template:\n%s", resolved_chat_template)
691
692
state.openai_serving_models = OpenAIServingModels(
693
+ engine_client=engine_client,
694
model_config=model_config,
695
base_model_paths=base_model_paths,
696
lora_modules=args.lora_modules,
697
prompt_adapters=args.prompt_adapters,
698
)
698
- # TODO: The chat template is now broken for lora adapters :(
699
+ await state.openai_serving_models.init_static_loras()
700
state.openai_serving_chat = OpenAIServingChat(
701
engine_client,
702
model_config,
@@ -794,7 +795,7 @@ async def run_server(args, **uvicorn_kwargs) -> None:
795
app = build_app(args)
796
797
model_config = await engine_client.get_model_config()
797
- init_app_state(engine_client, model_config, app.state, args)
798
+ await init_app_state(engine_client, model_config, app.state, args)
799
800
shutdown_task = await serve_http(
801
app,
vllm/entrypoints/openai/run_batch.py
+1
@@ -215,6 +215,7 @@ async def main(args):
215
216
# Create the openai serving objects.
217
openai_serving_models = OpenAIServingModels(
218
+ engine_client=engine,
219
model_config=model_config,
220
base_model_paths=base_model_paths,
221
lora_modules=None,
vllm/entrypoints/openai/serving_models.py
+59
-19
@@ -5,15 +5,19 @@ from http import HTTPStatus
5
from typing import List, Optional, Union
6
7
from vllm.config import ModelConfig
8
+from vllm.engine.protocol import EngineClient
9
from vllm.entrypoints.openai.protocol import (ErrorResponse,
10
LoadLoraAdapterRequest,
11
ModelCard, ModelList,
12
ModelPermission,
13
UnloadLoraAdapterRequest)
14
+from vllm.logger import init_logger
15
from vllm.lora.request import LoRARequest
16
from vllm.prompt_adapter.request import PromptAdapterRequest
17
from vllm.utils import AtomicCounter
18
19
+logger = init_logger(__name__)
20
+
21
22
@dataclass
23
class BaseModelPath:
@@ -45,6 +49,7 @@ class OpenAIServingModels:
49
50
def __init__(
51
self,
52
+ engine_client: EngineClient,
53
model_config: ModelConfig,
54
base_model_paths: List[BaseModelPath],
55
*,
@@ -55,20 +60,11 @@ class OpenAIServingModels:
60
61
self.base_model_paths = base_model_paths
62
self.max_model_len = model_config.max_model_len
63
+ self.engine_client = engine_client
64
65
+ self.static_lora_modules = lora_modules
66
+ self.lora_requests: List[LoRARequest] = []
67
self.lora_id_counter = AtomicCounter(0)
60
- self.lora_requests = []
61
- if lora_modules is not None:
62
- self.lora_requests = [
63
- LoRARequest(lora_name=lora.name,
64
- lora_int_id=i,
65
- lora_path=lora.path,
66
- base_model_name=lora.base_model_name
67
- if lora.base_model_name
68
- and self.is_base_model(lora.base_model_name) else
69
- self.base_model_paths[0].name)
70
- for i, lora in enumerate(lora_modules, start=1)
71
- ]
68
69
self.prompt_adapter_requests = []
70
if prompt_adapters is not None:
@@ -84,6 +80,19 @@ class OpenAIServingModels:
80
prompt_adapter_local_path=prompt_adapter.local_path,
81
prompt_adapter_num_virtual_tokens=num_virtual_tokens))
82
83
+ async def init_static_loras(self):
84
+ """Loads all static LoRA modules.
85
+ Raises if any fail to load"""
86
+ if self.static_lora_modules is None:
87
+ return
88
+ for lora in self.static_lora_modules:
89
+ load_request = LoadLoraAdapterRequest(lora_path=lora.path,
90
+ lora_name=lora.name)
91
+ load_result = await self.load_lora_adapter(
92
+ request=load_request, base_model_name=lora.base_model_name)
93
+ if isinstance(load_result, ErrorResponse):
94
+ raise ValueError(load_result.message)
95
+
96
def is_base_model(self, model_name):
97
return any(model.name == model_name for model in self.base_model_paths)
98
@@ -129,17 +138,47 @@ class OpenAIServingModels:
138
139
async def load_lora_adapter(
140
self,
132
- request: LoadLoraAdapterRequest) -> Union[ErrorResponse, str]:
141
+ request: LoadLoraAdapterRequest,
142
+ base_model_name: Optional[str] = None
143
+ ) -> Union[ErrorResponse, str]:
144
error_check_ret = await self._check_load_lora_adapter_request(request)
145
if error_check_ret is not None:
146
return error_check_ret
147
148
lora_name, lora_path = request.lora_name, request.lora_path
149
unique_id = self.lora_id_counter.inc(1)
139
- self.lora_requests.append(
140
- LoRARequest(lora_name=lora_name,
141
- lora_int_id=unique_id,
142
- lora_path=lora_path))
150
+ lora_request = LoRARequest(lora_name=lora_name,
151
+ lora_int_id=unique_id,
152
+ lora_path=lora_path)
153
+ if base_model_name is not None and self.is_base_model(base_model_name):
154
+ lora_request.base_model_name = base_model_name
155
+
156
+ # Validate that the adapter can be loaded into the engine
157
+ # This will also pre-load it for incoming requests
158
+ try:
159
+ await self.engine_client.add_lora(lora_request)
160
+ except ValueError as e:
161
+ # Adapter not found or lora configuration errors
162
+ if "No adapter found" in str(e):
163
+ return create_error_response(message=str(e),
164
+ err_type="NotFoundError",
165
+ status_code=HTTPStatus.NOT_FOUND)
166
+ else:
167
+ return create_error_response(
168
+ message=str(e),
169
+ err_type="BadRequestError",
170
+ status_code=HTTPStatus.BAD_REQUEST)
171
+ except BaseException as e:
172
+ # Some other unexpected problem loading the adapter, e.g. malformed
173
+ # input files.
174
+ # More detailed error messages for the user would be nicer here
175
+ return create_error_response(message=str(e),
176
+ err_type="BadRequestError",
177
+ status_code=HTTPStatus.BAD_REQUEST)
178
+
179
+ self.lora_requests.append(lora_request)
180
+ logger.info("Loaded new LoRA adapter: name '%s', path '%s'", lora_name,
181
+ lora_path)
182
return f"Success: LoRA adapter '{lora_name}' added successfully."
183
184
async def unload_lora_adapter(
@@ -155,6 +194,7 @@ class OpenAIServingModels:
194
lora_request for lora_request in self.lora_requests
195
if lora_request.lora_name != lora_name
196
]
197
+ logger.info("Removed LoRA adapter: name '%s'", lora_name)
198
return f"Success: LoRA adapter '{lora_name}' removed successfully."
199
200
async def _check_load_lora_adapter_request(
@@ -195,8 +235,8 @@ class OpenAIServingModels:
235
return create_error_response(
236
message=
237
f"The lora adapter '{request.lora_name}' cannot be found.",
198
- err_type="InvalidUserInput",
199
- status_code=HTTPStatus.BAD_REQUEST)
238
+ err_type="NotFoundError",
239
+ status_code=HTTPStatus.NOT_FOUND)
240
241
return None
242
vllm/lora/worker_manager.py
+17
-2
@@ -115,6 +115,14 @@ class WorkerLoRAManager(AbstractWorkerManager):
115
embedding_padding_modules=self.embedding_padding_modules,
116
weights_mapper=hf_to_vllm_mapper)
117
118
+ except FileNotFoundError as e:
119
+ # FileNotFoundError should be raised if both
120
+ # - No adapter found to download from huggingface (or in
121
+ # offline mode)
122
+ # - No local adapter files found at `lora_request.lora_path`
123
+ raise ValueError(
124
+ f"Loading lora {lora_request.lora_name} failed: No adapter "
125
+ f"found for {lora_path}") from e
126
except Exception as e:
127
raise RuntimeError(f"Loading lora {lora_path} failed") from e
128
if lora.rank > self.lora_config.max_lora_rank:
@@ -209,12 +217,19 @@ class LRUCacheWorkerLoRAManager(WorkerLoRAManager):
217
218
def add_adapter(self, lora_request: LoRARequest) -> bool:
219
if lora_request.lora_int_id not in self.list_adapters():
212
- # Remove before we load the new lora to save memory
220
+ # Load the new adapter first to ensure it is actually valid, before
221
+ # evicting any existing adapters.
222
+ # This may cause the # of loaded lora adapters to very temporarily
223
+ # exceed `--max-cpu-loras`.
224
+ lora = self._load_adapter(lora_request)
225
+
226
+ # Loading succeeded, now check if we will exceed cache capacity and
227
+ # evict if the oldest adapter if so
228
if len(self._adapter_manager) + 1 > self._adapter_manager.capacity:
229
assert isinstance(self._adapter_manager,
230
LRUCacheLoRAModelManager)
231
self._adapter_manager.remove_oldest_adapter()
217
- lora = self._load_adapter(lora_request)
232
+ # Then add the new adapter to the cache
233
loaded = self._adapter_manager.add_adapter(lora)
234
else:
235
# If the lora is already loaded, just touch it to
vllm/v1/engine/async_llm.py
+4
@@ -339,3 +339,7 @@ class AsyncLLM(EngineClient):
339
@property
340
def dead_error(self) -> BaseException:
341
return Exception() # TODO: implement
342
+
343
+ async def add_lora(self, lora_request: LoRARequest) -> None:
344
+ """Load a new LoRA adapter into the engine for future requests."""
345
+ raise NotImplementedError("LoRA not yet supported in V1")