@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ac2f3f7fee
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[Bugfix] Validate lora adapters to avoid crashing server (#11727)

Signed-off-by: Joe Runde <Joseph.Runde@ibm.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>

Joe Runde committed Jan 10, 2025 at 00:56 UTC ac2f3f7fee93cf9cd97c0078e362feab7b6c8299
15 files changed +459 -171
tests/entrypoints/openai/test_lora_adapters.py new
+269
@@ -0,0 +1,269 @@
1 +import asyncio
2 +import json
3 +import shutil
4 +from contextlib import suppress
5 +
6 +import openai # use the official client for correctness check
7 +import pytest
8 +import pytest_asyncio
9 +# downloading lora to test lora requests
10 +from huggingface_hub import snapshot_download
11 +
12 +from ...utils import RemoteOpenAIServer
13 +
14 +# any model with a chat template should work here
15 +MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
16 +# technically this needs Mistral-7B-v0.1 as base, but we're not testing
17 +# generation quality here
18 +LORA_NAME = "typeof/zephyr-7b-beta-lora"
19 +
20 +
21 +@pytest.fixture(scope="module")
22 +def zephyr_lora_files():
23 + return snapshot_download(repo_id=LORA_NAME)
24 +
25 +
26 +@pytest.fixture(scope="module")
27 +def server_with_lora_modules_json(zephyr_lora_files):
28 + # Define the json format LoRA module configurations
29 + lora_module_1 = {
30 + "name": "zephyr-lora",
31 + "path": zephyr_lora_files,
32 + "base_model_name": MODEL_NAME
33 + }
34 +
35 + lora_module_2 = {
36 + "name": "zephyr-lora2",
37 + "path": zephyr_lora_files,
38 + "base_model_name": MODEL_NAME
39 + }
40 +
41 + args = [
42 + # use half precision for speed and memory savings in CI environment
43 + "--dtype",
44 + "bfloat16",
45 + "--max-model-len",
46 + "8192",
47 + "--enforce-eager",
48 + # lora config below
49 + "--enable-lora",
50 + "--lora-modules",
51 + json.dumps(lora_module_1),
52 + json.dumps(lora_module_2),
53 + "--max-lora-rank",
54 + "64",
55 + "--max-cpu-loras",
56 + "2",
57 + "--max-num-seqs",
58 + "64",
59 + ]
60 +
61 + # Enable the /v1/load_lora_adapter endpoint
62 + envs = {"VLLM_ALLOW_RUNTIME_LORA_UPDATING": "True"}
63 +
64 + with RemoteOpenAIServer(MODEL_NAME, args, env_dict=envs) as remote_server:
65 + yield remote_server
66 +
67 +
68 +@pytest_asyncio.fixture
69 +async def client(server_with_lora_modules_json):
70 + async with server_with_lora_modules_json.get_async_client(
71 + ) as async_client:
72 + yield async_client
73 +
74 +
75 +@pytest.mark.asyncio
76 +async def test_static_lora_lineage(client: openai.AsyncOpenAI,
77 + zephyr_lora_files):
78 + models = await client.models.list()
79 + models = models.data
80 + served_model = models[0]
81 + lora_models = models[1:]
82 + assert served_model.id == MODEL_NAME
83 + assert served_model.root == MODEL_NAME
84 + assert served_model.parent is None
85 + assert all(lora_model.root == zephyr_lora_files
86 + for lora_model in lora_models)
87 + assert all(lora_model.parent == MODEL_NAME for lora_model in lora_models)
88 + assert lora_models[0].id == "zephyr-lora"
89 + assert lora_models[1].id == "zephyr-lora2"
90 +
91 +
92 +@pytest.mark.asyncio
93 +async def test_dynamic_lora_lineage(client: openai.AsyncOpenAI,
94 + zephyr_lora_files):
95 +
96 + response = await client.post("load_lora_adapter",
97 + cast_to=str,
98 + body={
99 + "lora_name": "zephyr-lora-3",
100 + "lora_path": zephyr_lora_files
101 + })
102 + # Ensure adapter loads before querying /models
103 + assert "success" in response
104 +
105 + models = await client.models.list()
106 + models = models.data
107 + dynamic_lora_model = models[-1]
108 + assert dynamic_lora_model.root == zephyr_lora_files
109 + assert dynamic_lora_model.parent == MODEL_NAME
110 + assert dynamic_lora_model.id == "zephyr-lora-3"
111 +
112 +
113 +@pytest.mark.asyncio
114 +async def test_dynamic_lora_not_found(client: openai.AsyncOpenAI):
115 + with pytest.raises(openai.NotFoundError):
116 + await client.post("load_lora_adapter",
117 + cast_to=str,
118 + body={
119 + "lora_name": "notfound",
120 + "lora_path": "/not/an/adapter"
121 + })
122 +
123 +
124 +@pytest.mark.asyncio
125 +async def test_dynamic_lora_invalid_files(client: openai.AsyncOpenAI,
126 + tmp_path):
127 + invalid_files = tmp_path / "invalid_files"
128 + invalid_files.mkdir()
129 + (invalid_files / "adapter_config.json").write_text("this is not json")
130 +
131 + with pytest.raises(openai.BadRequestError):
132 + await client.post("load_lora_adapter",
133 + cast_to=str,
134 + body={
135 + "lora_name": "invalid-json",
136 + "lora_path": str(invalid_files)
137 + })
138 +
139 +
140 +@pytest.mark.asyncio
141 +async def test_dynamic_lora_invalid_lora_rank(client: openai.AsyncOpenAI,
142 + tmp_path, zephyr_lora_files):
143 + invalid_rank = tmp_path / "invalid_rank"
144 +
145 + # Copy adapter from zephyr_lora_files to invalid_rank
146 + shutil.copytree(zephyr_lora_files, invalid_rank)
147 +
148 + with open(invalid_rank / "adapter_config.json") as f:
149 + adapter_config = json.load(f)
150 +
151 + print(adapter_config)
152 +
153 + # assert False
154 +
155 + # Change rank to invalid value
156 + adapter_config["r"] = 1024
157 + with open(invalid_rank / "adapter_config.json", "w") as f:
158 + json.dump(adapter_config, f)
159 +
160 + with pytest.raises(openai.BadRequestError,
161 + match="is greater than max_lora_rank"):
162 + await client.post("load_lora_adapter",
163 + cast_to=str,
164 + body={
165 + "lora_name": "invalid-json",
166 + "lora_path": str(invalid_rank)
167 + })
168 +
169 +
170 +@pytest.mark.asyncio
171 +async def test_multiple_lora_adapters(client: openai.AsyncOpenAI, tmp_path,
172 + zephyr_lora_files):
173 + """Validate that many loras can be dynamically registered and inferenced
174 + with concurrently"""
175 +
176 + # This test file configures the server with --max-cpu-loras=2 and this test
177 + # will concurrently load 10 adapters, so it should flex the LRU cache
178 + async def load_and_run_adapter(adapter_name: str):
179 + await client.post("load_lora_adapter",
180 + cast_to=str,
181 + body={
182 + "lora_name": adapter_name,
183 + "lora_path": str(zephyr_lora_files)
184 + })
185 + for _ in range(3):
186 + await client.completions.create(
187 + model=adapter_name,
188 + prompt=["Hello there", "Foo bar bazz buzz"],
189 + max_tokens=5,
190 + )
191 +
192 + lora_tasks = []
193 + for i in range(10):
194 + lora_tasks.append(
195 + asyncio.create_task(load_and_run_adapter(f"adapter_{i}")))
196 +
197 + results, _ = await asyncio.wait(lora_tasks)
198 +
199 + for r in results:
200 + assert not isinstance(r, Exception), f"Got exception {r}"
201 +
202 +
203 +@pytest.mark.asyncio
204 +async def test_loading_invalid_adapters_does_not_break_others(
205 + client: openai.AsyncOpenAI, tmp_path, zephyr_lora_files):
206 +
207 + invalid_files = tmp_path / "invalid_files"
208 + invalid_files.mkdir()
209 + (invalid_files / "adapter_config.json").write_text("this is not json")
210 +
211 + stop_good_requests_event = asyncio.Event()
212 +
213 + async def run_good_requests(client):
214 + # Run chat completions requests until event set
215 +
216 + results = []
217 +
218 + while not stop_good_requests_event.is_set():
219 + try:
220 + batch = await client.completions.create(
221 + model="zephyr-lora",
222 + prompt=["Hello there", "Foo bar bazz buzz"],
223 + max_tokens=5,
224 + )
225 + results.append(batch)
226 + except Exception as e:
227 + results.append(e)
228 +
229 + return results
230 +
231 + # Create task to run good requests
232 + good_task = asyncio.create_task(run_good_requests(client))
233 +
234 + # Run a bunch of bad adapter loads
235 + for _ in range(25):
236 + with suppress(openai.NotFoundError):
237 + await client.post("load_lora_adapter",
238 + cast_to=str,
239 + body={
240 + "lora_name": "notfound",
241 + "lora_path": "/not/an/adapter"
242 + })
243 + for _ in range(25):
244 + with suppress(openai.BadRequestError):
245 + await client.post("load_lora_adapter",
246 + cast_to=str,
247 + body={
248 + "lora_name": "invalid",
249 + "lora_path": str(invalid_files)
250 + })
251 +
252 + # Ensure all the running requests with lora adapters succeeded
253 + stop_good_requests_event.set()
254 + results = await good_task
255 + for r in results:
256 + assert not isinstance(r, Exception), f"Got exception {r}"
257 +
258 + # Ensure we can load another adapter and run it
259 + await client.post("load_lora_adapter",
260 + cast_to=str,
261 + body={
262 + "lora_name": "valid",
263 + "lora_path": zephyr_lora_files
264 + })
265 + await client.completions.create(
266 + model="valid",
267 + prompt=["Hello there", "Foo bar bazz buzz"],
268 + max_tokens=5,
269 + )
tests/entrypoints/openai/test_lora_lineage.py deleted
-109
@@ -1,109 +0,0 @@
1 -import json
2 -
3 -import openai # use the official client for correctness check
4 -import pytest
5 -import pytest_asyncio
6 -# downloading lora to test lora requests
7 -from huggingface_hub import snapshot_download
8 -
9 -from ...utils import RemoteOpenAIServer
10 -
11 -# any model with a chat template should work here
12 -MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
13 -# technically this needs Mistral-7B-v0.1 as base, but we're not testing
14 -# generation quality here
15 -LORA_NAME = "typeof/zephyr-7b-beta-lora"
16 -
17 -
18 -@pytest.fixture(scope="module")
19 -def zephyr_lora_files():
20 - return snapshot_download(repo_id=LORA_NAME)
21 -
22 -
23 -@pytest.fixture(scope="module")
24 -def server_with_lora_modules_json(zephyr_lora_files):
25 - # Define the json format LoRA module configurations
26 - lora_module_1 = {
27 - "name": "zephyr-lora",
28 - "path": zephyr_lora_files,
29 - "base_model_name": MODEL_NAME
30 - }
31 -
32 - lora_module_2 = {
33 - "name": "zephyr-lora2",
34 - "path": zephyr_lora_files,
35 - "base_model_name": MODEL_NAME
36 - }
37 -
38 - args = [
39 - # use half precision for speed and memory savings in CI environment
40 - "--dtype",
41 - "bfloat16",
42 - "--max-model-len",
43 - "8192",
44 - "--enforce-eager",
45 - # lora config below
46 - "--enable-lora",
47 - "--lora-modules",
48 - json.dumps(lora_module_1),
49 - json.dumps(lora_module_2),
50 - "--max-lora-rank",
51 - "64",
52 - "--max-cpu-loras",
53 - "2",
54 - "--max-num-seqs",
55 - "64",
56 - ]
57 -
58 - # Enable the /v1/load_lora_adapter endpoint
59 - envs = {"VLLM_ALLOW_RUNTIME_LORA_UPDATING": "True"}
60 -
61 - with RemoteOpenAIServer(MODEL_NAME, args, env_dict=envs) as remote_server:
62 - yield remote_server
63 -
64 -
65 -@pytest_asyncio.fixture
66 -async def client_for_lora_lineage(server_with_lora_modules_json):
67 - async with server_with_lora_modules_json.get_async_client(
68 - ) as async_client:
69 - yield async_client
70 -
71 -
72 -@pytest.mark.asyncio
73 -async def test_static_lora_lineage(client_for_lora_lineage: openai.AsyncOpenAI,
74 - zephyr_lora_files):
75 - models = await client_for_lora_lineage.models.list()
76 - models = models.data
77 - served_model = models[0]
78 - lora_models = models[1:]
79 - assert served_model.id == MODEL_NAME
80 - assert served_model.root == MODEL_NAME
81 - assert served_model.parent is None
82 - assert all(lora_model.root == zephyr_lora_files
83 - for lora_model in lora_models)
84 - assert all(lora_model.parent == MODEL_NAME for lora_model in lora_models)
85 - assert lora_models[0].id == "zephyr-lora"
86 - assert lora_models[1].id == "zephyr-lora2"
87 -
88 -
89 -@pytest.mark.asyncio
90 -async def test_dynamic_lora_lineage(
91 - client_for_lora_lineage: openai.AsyncOpenAI, zephyr_lora_files):
92 -
93 - response = await client_for_lora_lineage.post("load_lora_adapter",
94 - cast_to=str,
95 - body={
96 - "lora_name":
97 - "zephyr-lora-3",
98 - "lora_path":
99 - zephyr_lora_files
100 - })
101 - # Ensure adapter loads before querying /models
102 - assert "success" in response
103 -
104 - models = await client_for_lora_lineage.models.list()
105 - models = models.data
106 - dynamic_lora_model = models[-1]
107 - assert dynamic_lora_model.root == zephyr_lora_files
108 - assert dynamic_lora_model.parent == MODEL_NAME
109 - assert dynamic_lora_model.id == "zephyr-lora-3"
tests/entrypoints/openai/test_serving_chat.py
+5 -3
@@ -52,7 +52,7 @@ async def _async_serving_chat_init():
52 engine = MockEngine()
53 model_config = await engine.get_model_config()
54
55 - models = OpenAIServingModels(model_config, BASE_MODEL_PATHS)
55 + models = OpenAIServingModels(engine, model_config, BASE_MODEL_PATHS)
56 serving_completion = OpenAIServingChat(engine,
57 model_config,
58 models,
@@ -73,7 +73,8 @@ def test_serving_chat_should_set_correct_max_tokens():
73 mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME)
74 mock_engine.errored = False
75
76 - models = OpenAIServingModels(base_model_paths=BASE_MODEL_PATHS,
76 + models = OpenAIServingModels(engine_client=mock_engine,
77 + base_model_paths=BASE_MODEL_PATHS,
78 model_config=MockModelConfig())
79 serving_chat = OpenAIServingChat(mock_engine,
80 MockModelConfig(),
@@ -116,7 +117,8 @@ def test_serving_chat_could_load_correct_generation_config():
117 mock_engine.errored = False
118
119 # Initialize the serving chat
119 - models = OpenAIServingModels(base_model_paths=BASE_MODEL_PATHS,
120 + models = OpenAIServingModels(engine_client=mock_engine,
121 + base_model_paths=BASE_MODEL_PATHS,
122 model_config=mock_model_config)
123 serving_chat = OpenAIServingChat(mock_engine,
124 mock_model_config,
tests/entrypoints/openai/test_serving_models.py
+7 -3
@@ -4,6 +4,7 @@ from unittest.mock import MagicMock
4 import pytest
5
6 from vllm.config import ModelConfig
7 +from vllm.engine.protocol import EngineClient
8 from vllm.entrypoints.openai.protocol import (ErrorResponse,
9 LoadLoraAdapterRequest,
10 UnloadLoraAdapterRequest)
@@ -21,13 +22,16 @@ LORA_UNLOADING_SUCCESS_MESSAGE = (
22
23 async def _async_serving_models_init() -> OpenAIServingModels:
24 mock_model_config = MagicMock(spec=ModelConfig)
25 + mock_engine_client = MagicMock(spec=EngineClient)
26 # Set the max_model_len attribute to avoid missing attribute
27 mock_model_config.max_model_len = 2048
28
27 - serving_models = OpenAIServingModels(base_model_paths=BASE_MODEL_PATHS,
29 + serving_models = OpenAIServingModels(engine_client=mock_engine_client,
30 + base_model_paths=BASE_MODEL_PATHS,
31 model_config=mock_model_config,
32 lora_modules=None,
33 prompt_adapters=None)
34 + await serving_models.init_static_loras()
35
36 return serving_models
37
@@ -113,5 +117,5 @@ async def test_unload_lora_adapter_not_found():
117 request = UnloadLoraAdapterRequest(lora_name="nonexistent_adapter")
118 response = await serving_models.unload_lora_adapter(request)
119 assert isinstance(response, ErrorResponse)
116 - assert response.type == "InvalidUserInput"
117 - assert response.code == HTTPStatus.BAD_REQUEST
120 + assert response.type == "NotFoundError"
121 + assert response.code == HTTPStatus.NOT_FOUND
tests/entrypoints/openai/test_shutdown.py
+8 -19
@@ -1,6 +1,3 @@
1 -import json
2 -import os
3 -
1 import openai
2 import pytest
3
@@ -10,16 +7,7 @@ MODEL_NAME = "meta-llama/Llama-3.2-1B"
7
8
9 @pytest.mark.asyncio
13 -async def test_shutdown_on_engine_failure(tmp_path):
14 - # Use a bad adapter to crash the engine
15 - # (This test will fail when that bug is fixed)
16 - adapter_path = tmp_path / "bad_adapter"
17 - os.mkdir(adapter_path)
18 - with open(adapter_path / "adapter_model_config.json", "w") as f:
19 - json.dump({"not": "real"}, f)
20 - with open(adapter_path / "adapter_model.safetensors", "wb") as f:
21 - f.write(b"this is fake")
22 -
10 +async def test_shutdown_on_engine_failure():
11 # dtype, max-len etc set so that this can run in CI
12 args = [
13 "--dtype",
@@ -29,9 +17,6 @@ async def test_shutdown_on_engine_failure(tmp_path):
17 "--enforce-eager",
18 "--max-num-seqs",
19 "128",
32 - "--enable-lora",
33 - "--lora-modules",
34 - f"bad-adapter={tmp_path / 'bad_adapter'}",
20 ]
21
22 with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
@@ -39,9 +24,13 @@ async def test_shutdown_on_engine_failure(tmp_path):
24
25 with pytest.raises(
26 (openai.APIConnectionError, openai.InternalServerError)):
42 - # This crashes the engine
43 - await client.completions.create(model="bad-adapter",
44 - prompt="Hello, my name is")
27 + # Asking for lots of prompt logprobs will currently crash the
28 + # engine. This may change in the future when that bug is fixed
29 + prompt = "Hello " * 4000
30 + await client.completions.create(
31 + model=MODEL_NAME,
32 + prompt=prompt,
33 + extra_body={"prompt_logprobs": 10})
34
35 # Now the server should shut down
36 return_code = remote_server.proc.wait(timeout=8)
vllm/engine/async_llm_engine.py
+4
@@ -1257,6 +1257,10 @@ class AsyncLLMEngine(EngineClient):
1257 else:
1258 self.engine.model_executor._run_workers("stop_profile")
1259
1260 + async def add_lora(self, lora_request: LoRARequest) -> None:
1261 + """Load a new LoRA adapter into the engine for future requests."""
1262 + self.engine.add_lora(lora_request)
1263 +
1264
1265 # TODO(v1): Remove this class proxy when V1 goes default.
1266 if envs.VLLM_USE_V1:
vllm/engine/multiprocessing/__init__.py
+17 -3
@@ -1,4 +1,5 @@
1 -from dataclasses import dataclass
1 +import uuid
2 +from dataclasses import dataclass, field
3 from enum import Enum
4 from typing import List, Mapping, Optional, Union, overload
5
@@ -120,10 +121,23 @@ class RPCUProfileRequest(Enum):
121 STOP_PROFILE = 2
122
123
124 +@dataclass
125 +class RPCLoadAdapterRequest:
126 + lora_request: LoRARequest
127 + # Set the default value of request_id to a new UUID
128 + request_id: str = field(default_factory=lambda: str(uuid.uuid4()))
129 +
130 +
131 +@dataclass
132 +class RPCAdapterLoadedResponse:
133 + request_id: str
134 +
135 +
136 RPC_REQUEST_T = Union[RPCProcessRequest, RPCAbortRequest, RPCStartupRequest,
124 - RPCUProfileRequest]
137 + RPCUProfileRequest, RPCLoadAdapterRequest]
138
126 -REQUEST_OUTPUTS_T = Union[List[RequestOutput], RPCError]
139 +REQUEST_OUTPUTS_T = Union[List[RequestOutput], RPCAdapterLoadedResponse,
140 + RPCError]
141
142
143 def ENGINE_DEAD_ERROR(
vllm/engine/multiprocessing/client.py
+35 -7
@@ -25,8 +25,10 @@ from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT,
25 IPC_HEALTH_EXT, IPC_INPUT_EXT,
26 IPC_OUTPUT_EXT, RPC_REQUEST_T,
27 VLLM_RPC_SUCCESS_STR, RPCAbortRequest,
28 - RPCError, RPCProcessRequest,
29 - RPCStartupRequest, RPCStartupResponse,
28 + RPCAdapterLoadedResponse, RPCError,
29 + RPCLoadAdapterRequest,
30 + RPCProcessRequest, RPCStartupRequest,
31 + RPCStartupResponse,
32 RPCUProfileRequest)
33 from vllm.engine.protocol import EngineClient
34 # yapf: enable
@@ -240,17 +242,22 @@ class MQLLMEngineClient(EngineClient):
242 queue = self.output_queues.get(request_id)
243 if queue is not None:
244 queue.put_nowait(exception)
245 + # Put each output into the appropriate queue.
246 + elif isinstance(request_outputs, RPCAdapterLoadedResponse):
247 + self._add_output(request_outputs)
248 else:
244 - # Put each output into the appropriate steam.
249 for request_output in request_outputs:
246 - queue = self.output_queues.get(
247 - request_output.request_id)
248 - if queue is not None:
249 - queue.put_nowait(request_output)
250 + self._add_output(request_output)
251
252 except asyncio.CancelledError:
253 logger.debug("Shutting down MQLLMEngineClient output handler.")
254
255 + def _add_output(self, request_output: Union[RequestOutput,
256 + RPCAdapterLoadedResponse]):
257 + queue = self.output_queues.get(request_output.request_id)
258 + if queue is not None:
259 + queue.put_nowait(request_output)
260 +
261 async def setup(self):
262 """Setup the client before it starts sending server requests."""
263
@@ -659,3 +666,24 @@ class MQLLMEngineClient(EngineClient):
666
667 await self._send_one_way_rpc_request(
668 request=RPCUProfileRequest.STOP_PROFILE, socket=self.input_socket)
669 +
670 + async def add_lora(self, lora_request: LoRARequest) -> None:
671 + """Load a new LoRA adapter into the engine for future requests."""
672 + # Uses the same I/O as generate requests
673 + request = RPCLoadAdapterRequest(lora_request)
674 +
675 + # Create output queue for this requests.
676 + queue: asyncio.Queue[Union[None, BaseException]] = asyncio.Queue()
677 + self.output_queues[request.request_id] = queue
678 +
679 + # Send the request
680 + request_bytes = pickle.dumps(request)
681 + await self.input_socket.send_multipart((request_bytes, ), copy=False)
682 +
683 + # Wait for the response
684 + request_output = await queue.get()
685 + self.output_queues.pop(request.request_id)
686 +
687 + # Raise on error, otherwise happily return None
688 + if isinstance(request_output, BaseException):
689 + raise request_output
vllm/engine/multiprocessing/engine.py
+24 -3
@@ -14,8 +14,10 @@ from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT,
14 IPC_HEALTH_EXT, IPC_INPUT_EXT,
15 IPC_OUTPUT_EXT, REQUEST_OUTPUTS_T,
16 VLLM_RPC_SUCCESS_STR, RPCAbortRequest,
17 - RPCError, RPCProcessRequest,
18 - RPCStartupRequest, RPCStartupResponse,
17 + RPCAdapterLoadedResponse, RPCError,
18 + RPCLoadAdapterRequest,
19 + RPCProcessRequest, RPCStartupRequest,
20 + RPCStartupResponse,
21 RPCUProfileRequest)
22 # yapf: enable
23 from vllm.executor.gpu_executor import GPUExecutor
@@ -234,6 +236,8 @@ class MQLLMEngine:
236 self.start_profile()
237 else:
238 self.stop_profile()
239 + elif isinstance(request, RPCLoadAdapterRequest):
240 + self._handle_load_adapter_request(request)
241 else:
242 raise ValueError("Unknown RPCRequest Type: "
243 f"{type(request)}")
@@ -284,6 +288,19 @@ class MQLLMEngine:
288 if self.log_requests:
289 logger.info("Aborted request %s.", request.request_id)
290
291 + def _handle_load_adapter_request(self, request: RPCLoadAdapterRequest):
292 + try:
293 + self.engine.add_lora(request.lora_request)
294 + except BaseException as e:
295 + # Send back an error if the adater fails to load
296 + rpc_err = RPCError(request_id=request.request_id,
297 + is_engine_errored=False,
298 + exception=e)
299 + self._send_outputs(rpc_err)
300 + # Otherwise, send back the successful load message
301 + self._send_outputs(
302 + RPCAdapterLoadedResponse(request_id=request.request_id))
303 +
304 def _health_check(self):
305 # Send unhealthy if engine has already errored
306 if self._errored_with is not None:
@@ -296,7 +313,11 @@ class MQLLMEngine:
313 self._send_unhealthy(e)
314
315 def _send_outputs(self, outputs: REQUEST_OUTPUTS_T):
299 - """Send List of RequestOutput to RPCClient."""
316 + """Send outputs back to the engine client. These can be:
317 + - Exceptions
318 + - A list of generation outputs
319 + - A response from loading a lora adapter
320 + """
321 if outputs:
322 try:
323 from ray.exceptions import RayTaskError
vllm/engine/protocol.py
+5
@@ -270,3 +270,8 @@ class EngineClient(ABC):
270 async def stop_profile(self) -> None:
271 """Start profiling the engine"""
272 ...
273 +
274 + @abstractmethod
275 + async def add_lora(self, lora_request: LoRARequest) -> None:
276 + """Load a new LoRA adapter into the engine for future requests."""
277 + ...
vllm/entrypoints/openai/api_server.py
+4 -3
@@ -662,7 +662,7 @@ def build_app(args: Namespace) -> FastAPI:
662 return app
663
664
665 -def init_app_state(
665 +async def init_app_state(
666 engine_client: EngineClient,
667 model_config: ModelConfig,
668 state: State,
@@ -690,12 +690,13 @@ def init_app_state(
690 logger.info("Using supplied chat template:\n%s", resolved_chat_template)
691
692 state.openai_serving_models = OpenAIServingModels(
693 + engine_client=engine_client,
694 model_config=model_config,
695 base_model_paths=base_model_paths,
696 lora_modules=args.lora_modules,
697 prompt_adapters=args.prompt_adapters,
698 )
698 - # TODO: The chat template is now broken for lora adapters :(
699 + await state.openai_serving_models.init_static_loras()
700 state.openai_serving_chat = OpenAIServingChat(
701 engine_client,
702 model_config,
@@ -794,7 +795,7 @@ async def run_server(args, **uvicorn_kwargs) -> None:
795 app = build_app(args)
796
797 model_config = await engine_client.get_model_config()
797 - init_app_state(engine_client, model_config, app.state, args)
798 + await init_app_state(engine_client, model_config, app.state, args)
799
800 shutdown_task = await serve_http(
801 app,
vllm/entrypoints/openai/run_batch.py
+1
@@ -215,6 +215,7 @@ async def main(args):
215
216 # Create the openai serving objects.
217 openai_serving_models = OpenAIServingModels(
218 + engine_client=engine,
219 model_config=model_config,
220 base_model_paths=base_model_paths,
221 lora_modules=None,
vllm/entrypoints/openai/serving_models.py
+59 -19
@@ -5,15 +5,19 @@ from http import HTTPStatus
5 from typing import List, Optional, Union
6
7 from vllm.config import ModelConfig
8 +from vllm.engine.protocol import EngineClient
9 from vllm.entrypoints.openai.protocol import (ErrorResponse,
10 LoadLoraAdapterRequest,
11 ModelCard, ModelList,
12 ModelPermission,
13 UnloadLoraAdapterRequest)
14 +from vllm.logger import init_logger
15 from vllm.lora.request import LoRARequest
16 from vllm.prompt_adapter.request import PromptAdapterRequest
17 from vllm.utils import AtomicCounter
18
19 +logger = init_logger(__name__)
20 +
21
22 @dataclass
23 class BaseModelPath:
@@ -45,6 +49,7 @@ class OpenAIServingModels:
49
50 def __init__(
51 self,
52 + engine_client: EngineClient,
53 model_config: ModelConfig,
54 base_model_paths: List[BaseModelPath],
55 *,
@@ -55,20 +60,11 @@ class OpenAIServingModels:
60
61 self.base_model_paths = base_model_paths
62 self.max_model_len = model_config.max_model_len
63 + self.engine_client = engine_client
64
65 + self.static_lora_modules = lora_modules
66 + self.lora_requests: List[LoRARequest] = []
67 self.lora_id_counter = AtomicCounter(0)
60 - self.lora_requests = []
61 - if lora_modules is not None:
62 - self.lora_requests = [
63 - LoRARequest(lora_name=lora.name,
64 - lora_int_id=i,
65 - lora_path=lora.path,
66 - base_model_name=lora.base_model_name
67 - if lora.base_model_name
68 - and self.is_base_model(lora.base_model_name) else
69 - self.base_model_paths[0].name)
70 - for i, lora in enumerate(lora_modules, start=1)
71 - ]
68
69 self.prompt_adapter_requests = []
70 if prompt_adapters is not None:
@@ -84,6 +80,19 @@ class OpenAIServingModels:
80 prompt_adapter_local_path=prompt_adapter.local_path,
81 prompt_adapter_num_virtual_tokens=num_virtual_tokens))
82
83 + async def init_static_loras(self):
84 + """Loads all static LoRA modules.
85 + Raises if any fail to load"""
86 + if self.static_lora_modules is None:
87 + return
88 + for lora in self.static_lora_modules:
89 + load_request = LoadLoraAdapterRequest(lora_path=lora.path,
90 + lora_name=lora.name)
91 + load_result = await self.load_lora_adapter(
92 + request=load_request, base_model_name=lora.base_model_name)
93 + if isinstance(load_result, ErrorResponse):
94 + raise ValueError(load_result.message)
95 +
96 def is_base_model(self, model_name):
97 return any(model.name == model_name for model in self.base_model_paths)
98
@@ -129,17 +138,47 @@ class OpenAIServingModels:
138
139 async def load_lora_adapter(
140 self,
132 - request: LoadLoraAdapterRequest) -> Union[ErrorResponse, str]:
141 + request: LoadLoraAdapterRequest,
142 + base_model_name: Optional[str] = None
143 + ) -> Union[ErrorResponse, str]:
144 error_check_ret = await self._check_load_lora_adapter_request(request)
145 if error_check_ret is not None:
146 return error_check_ret
147
148 lora_name, lora_path = request.lora_name, request.lora_path
149 unique_id = self.lora_id_counter.inc(1)
139 - self.lora_requests.append(
140 - LoRARequest(lora_name=lora_name,
141 - lora_int_id=unique_id,
142 - lora_path=lora_path))
150 + lora_request = LoRARequest(lora_name=lora_name,
151 + lora_int_id=unique_id,
152 + lora_path=lora_path)
153 + if base_model_name is not None and self.is_base_model(base_model_name):
154 + lora_request.base_model_name = base_model_name
155 +
156 + # Validate that the adapter can be loaded into the engine
157 + # This will also pre-load it for incoming requests
158 + try:
159 + await self.engine_client.add_lora(lora_request)
160 + except ValueError as e:
161 + # Adapter not found or lora configuration errors
162 + if "No adapter found" in str(e):
163 + return create_error_response(message=str(e),
164 + err_type="NotFoundError",
165 + status_code=HTTPStatus.NOT_FOUND)
166 + else:
167 + return create_error_response(
168 + message=str(e),
169 + err_type="BadRequestError",
170 + status_code=HTTPStatus.BAD_REQUEST)
171 + except BaseException as e:
172 + # Some other unexpected problem loading the adapter, e.g. malformed
173 + # input files.
174 + # More detailed error messages for the user would be nicer here
175 + return create_error_response(message=str(e),
176 + err_type="BadRequestError",
177 + status_code=HTTPStatus.BAD_REQUEST)
178 +
179 + self.lora_requests.append(lora_request)
180 + logger.info("Loaded new LoRA adapter: name '%s', path '%s'", lora_name,
181 + lora_path)
182 return f"Success: LoRA adapter '{lora_name}' added successfully."
183
184 async def unload_lora_adapter(
@@ -155,6 +194,7 @@ class OpenAIServingModels:
194 lora_request for lora_request in self.lora_requests
195 if lora_request.lora_name != lora_name
196 ]
197 + logger.info("Removed LoRA adapter: name '%s'", lora_name)
198 return f"Success: LoRA adapter '{lora_name}' removed successfully."
199
200 async def _check_load_lora_adapter_request(
@@ -195,8 +235,8 @@ class OpenAIServingModels:
235 return create_error_response(
236 message=
237 f"The lora adapter '{request.lora_name}' cannot be found.",
198 - err_type="InvalidUserInput",
199 - status_code=HTTPStatus.BAD_REQUEST)
238 + err_type="NotFoundError",
239 + status_code=HTTPStatus.NOT_FOUND)
240
241 return None
242
vllm/lora/worker_manager.py
+17 -2
@@ -115,6 +115,14 @@ class WorkerLoRAManager(AbstractWorkerManager):
115 embedding_padding_modules=self.embedding_padding_modules,
116 weights_mapper=hf_to_vllm_mapper)
117
118 + except FileNotFoundError as e:
119 + # FileNotFoundError should be raised if both
120 + # - No adapter found to download from huggingface (or in
121 + # offline mode)
122 + # - No local adapter files found at `lora_request.lora_path`
123 + raise ValueError(
124 + f"Loading lora {lora_request.lora_name} failed: No adapter "
125 + f"found for {lora_path}") from e
126 except Exception as e:
127 raise RuntimeError(f"Loading lora {lora_path} failed") from e
128 if lora.rank > self.lora_config.max_lora_rank:
@@ -209,12 +217,19 @@ class LRUCacheWorkerLoRAManager(WorkerLoRAManager):
217
218 def add_adapter(self, lora_request: LoRARequest) -> bool:
219 if lora_request.lora_int_id not in self.list_adapters():
212 - # Remove before we load the new lora to save memory
220 + # Load the new adapter first to ensure it is actually valid, before
221 + # evicting any existing adapters.
222 + # This may cause the # of loaded lora adapters to very temporarily
223 + # exceed `--max-cpu-loras`.
224 + lora = self._load_adapter(lora_request)
225 +
226 + # Loading succeeded, now check if we will exceed cache capacity and
227 + # evict if the oldest adapter if so
228 if len(self._adapter_manager) + 1 > self._adapter_manager.capacity:
229 assert isinstance(self._adapter_manager,
230 LRUCacheLoRAModelManager)
231 self._adapter_manager.remove_oldest_adapter()
217 - lora = self._load_adapter(lora_request)
232 + # Then add the new adapter to the cache
233 loaded = self._adapter_manager.add_adapter(lora)
234 else:
235 # If the lora is already loaded, just touch it to
vllm/v1/engine/async_llm.py
+4
@@ -339,3 +339,7 @@ class AsyncLLM(EngineClient):
339 @property
340 def dead_error(self) -> BaseException:
341 return Exception() # TODO: implement
342 +
343 + async def add_lora(self, lora_request: LoRARequest) -> None:
344 + """Load a new LoRA adapter into the engine for future requests."""
345 + raise NotImplementedError("LoRA not yet supported in V1")