A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Core] Support load and unload LoRA in api server (#6566)
Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
Jiaxin Shan committed
Sep 5, 2024 at 18:10 UTC
db3bf7c991cd1a0297d1a8ba501e59cfa226c337
10 files changed
+336
-6
docs/requirements-docs.txt
-1
@@ -11,6 +11,5 @@ pydantic >= 2.8
11
torch
12
py-cpuinfo
13
transformers
14
-openai # Required by docs/source/serving/openai_compatible_server.md's vllm.entrypoints.openai.cli_args
14
mistral_common >= 1.3.4
15
openai # Required by docs/source/serving/openai_compatible_server.md's vllm.entrypoints.openai.cli_args
\ No newline at end of file
docs/source/models/lora.rst
+52
@@ -107,3 +107,55 @@ The following is an example request
107
"max_tokens": 7,
108
"temperature": 0
109
}' | jq
110
+
111
+
112
+Dynamically serving LoRA Adapters
113
+---------------------------------
114
+
115
+In addition to serving LoRA adapters at server startup, the vLLM server now supports dynamically loading and unloading
116
+LoRA adapters at runtime through dedicated API endpoints. This feature can be particularly useful when the flexibility
117
+to change models on-the-fly is needed.
118
+
119
+Note: Enabling this feature in production environments is risky as user may participate model adapter management.
120
+
121
+To enable dynamic LoRA loading and unloading, ensure that the environment variable `VLLM_ALLOW_RUNTIME_LORA_UPDATING`
122
+is set to `True`. When this option is enabled, the API server will log a warning to indicate that dynamic loading is active.
123
+
124
+.. code-block:: bash
125
+
126
+ export VLLM_ALLOW_RUNTIME_LORA_UPDATING=True
127
+
128
+
129
+Loading a LoRA Adapter:
130
+
131
+To dynamically load a LoRA adapter, send a POST request to the `/v1/load_lora_adapter` endpoint with the necessary
132
+details of the adapter to be loaded. The request payload should include the name and path to the LoRA adapter.
133
+
134
+Example request to load a LoRA adapter:
135
+
136
+.. code-block:: bash
137
+
138
+ curl -X POST http://localhost:8000/v1/load_lora_adapter \
139
+ -H "Content-Type: application/json" \
140
+ -d '{
141
+ "lora_name": "sql_adapter",
142
+ "lora_path": "/path/to/sql-lora-adapter"
143
+ }'
144
+
145
+Upon a successful request, the API will respond with a 200 OK status code. If an error occurs, such as if the adapter
146
+cannot be found or loaded, an appropriate error message will be returned.
147
+
148
+Unloading a LoRA Adapter:
149
+
150
+To unload a LoRA adapter that has been previously loaded, send a POST request to the `/v1/unload_lora_adapter` endpoint
151
+with the name or ID of the adapter to be unloaded.
152
+
153
+Example request to unload a LoRA adapter:
154
+
155
+.. code-block:: bash
156
+
157
+ curl -X POST http://localhost:8000/v1/unload_lora_adapter \
158
+ -H "Content-Type: application/json" \
159
+ -d '{
160
+ "lora_name": "sql_adapter"
161
+ }'
tests/entrypoints/llm/test_generate_multiple_loras.py
+1
-1
@@ -50,7 +50,7 @@ def zephyr_lora_files():
50
@pytest.mark.skip_global_cleanup
51
def test_multiple_lora_requests(llm: LLM, zephyr_lora_files):
52
lora_request = [
53
- LoRARequest(LORA_NAME, idx + 1, zephyr_lora_files)
53
+ LoRARequest(LORA_NAME + str(idx), idx + 1, zephyr_lora_files)
54
for idx in range(len(PROMPTS))
55
]
56
# Multiple SamplingParams should be matched with each prompt
tests/entrypoints/openai/test_serving_engine.py
new
+107
@@ -0,0 +1,107 @@
1
+from http import HTTPStatus
2
+from unittest.mock import MagicMock
3
+
4
+import pytest
5
+
6
+from vllm.config import ModelConfig
7
+from vllm.engine.protocol import AsyncEngineClient
8
+from vllm.entrypoints.openai.protocol import (ErrorResponse,
9
+ LoadLoraAdapterRequest,
10
+ UnloadLoraAdapterRequest)
11
+from vllm.entrypoints.openai.serving_engine import OpenAIServing
12
+
13
+MODEL_NAME = "meta-llama/Llama-2-7b"
14
+LORA_LOADING_SUCCESS_MESSAGE = (
15
+ "Success: LoRA adapter '{lora_name}' added successfully.")
16
+LORA_UNLOADING_SUCCESS_MESSAGE = (
17
+ "Success: LoRA adapter '{lora_name}' removed successfully.")
18
+
19
+
20
+async def _async_serving_engine_init():
21
+ mock_engine_client = MagicMock(spec=AsyncEngineClient)
22
+ mock_model_config = MagicMock(spec=ModelConfig)
23
+ # Set the max_model_len attribute to avoid missing attribute
24
+ mock_model_config.max_model_len = 2048
25
+
26
+ serving_engine = OpenAIServing(mock_engine_client,
27
+ mock_model_config,
28
+ served_model_names=[MODEL_NAME],
29
+ lora_modules=None,
30
+ prompt_adapters=None,
31
+ request_logger=None)
32
+ return serving_engine
33
+
34
+
35
+@pytest.mark.asyncio
36
+async def test_load_lora_adapter_success():
37
+ serving_engine = await _async_serving_engine_init()
38
+ request = LoadLoraAdapterRequest(lora_name="adapter",
39
+ lora_path="/path/to/adapter2")
40
+ response = await serving_engine.load_lora_adapter(request)
41
+ assert response == LORA_LOADING_SUCCESS_MESSAGE.format(lora_name='adapter')
42
+ assert len(serving_engine.lora_requests) == 1
43
+ assert serving_engine.lora_requests[0].lora_name == "adapter"
44
+
45
+
46
+@pytest.mark.asyncio
47
+async def test_load_lora_adapter_missing_fields():
48
+ serving_engine = await _async_serving_engine_init()
49
+ request = LoadLoraAdapterRequest(lora_name="", lora_path="")
50
+ response = await serving_engine.load_lora_adapter(request)
51
+ assert isinstance(response, ErrorResponse)
52
+ assert response.type == "InvalidUserInput"
53
+ assert response.code == HTTPStatus.BAD_REQUEST
54
+
55
+
56
+@pytest.mark.asyncio
57
+async def test_load_lora_adapter_duplicate():
58
+ serving_engine = await _async_serving_engine_init()
59
+ request = LoadLoraAdapterRequest(lora_name="adapter1",
60
+ lora_path="/path/to/adapter1")
61
+ response = await serving_engine.load_lora_adapter(request)
62
+ assert response == LORA_LOADING_SUCCESS_MESSAGE.format(
63
+ lora_name='adapter1')
64
+ assert len(serving_engine.lora_requests) == 1
65
+
66
+ request = LoadLoraAdapterRequest(lora_name="adapter1",
67
+ lora_path="/path/to/adapter1")
68
+ response = await serving_engine.load_lora_adapter(request)
69
+ assert isinstance(response, ErrorResponse)
70
+ assert response.type == "InvalidUserInput"
71
+ assert response.code == HTTPStatus.BAD_REQUEST
72
+ assert len(serving_engine.lora_requests) == 1
73
+
74
+
75
+@pytest.mark.asyncio
76
+async def test_unload_lora_adapter_success():
77
+ serving_engine = await _async_serving_engine_init()
78
+ request = LoadLoraAdapterRequest(lora_name="adapter1",
79
+ lora_path="/path/to/adapter1")
80
+ response = await serving_engine.load_lora_adapter(request)
81
+ assert len(serving_engine.lora_requests) == 1
82
+
83
+ request = UnloadLoraAdapterRequest(lora_name="adapter1")
84
+ response = await serving_engine.unload_lora_adapter(request)
85
+ assert response == LORA_UNLOADING_SUCCESS_MESSAGE.format(
86
+ lora_name='adapter1')
87
+ assert len(serving_engine.lora_requests) == 0
88
+
89
+
90
+@pytest.mark.asyncio
91
+async def test_unload_lora_adapter_missing_fields():
92
+ serving_engine = await _async_serving_engine_init()
93
+ request = UnloadLoraAdapterRequest(lora_name="", lora_int_id=None)
94
+ response = await serving_engine.unload_lora_adapter(request)
95
+ assert isinstance(response, ErrorResponse)
96
+ assert response.type == "InvalidUserInput"
97
+ assert response.code == HTTPStatus.BAD_REQUEST
98
+
99
+
100
+@pytest.mark.asyncio
101
+async def test_unload_lora_adapter_not_found():
102
+ serving_engine = await _async_serving_engine_init()
103
+ request = UnloadLoraAdapterRequest(lora_name="nonexistent_adapter")
104
+ response = await serving_engine.unload_lora_adapter(request)
105
+ assert isinstance(response, ErrorResponse)
106
+ assert response.type == "InvalidUserInput"
107
+ assert response.code == HTTPStatus.BAD_REQUEST
vllm/entrypoints/openai/api_server.py
+38
-2
@@ -35,11 +35,13 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
35
DetokenizeResponse,
36
EmbeddingRequest,
37
EmbeddingResponse, ErrorResponse,
38
+ LoadLoraAdapterRequest,
39
TokenizeRequest,
39
- TokenizeResponse)
40
-# yapf: enable
40
+ TokenizeResponse,
41
+ UnloadLoraAdapterRequest)
42
from vllm.entrypoints.openai.rpc.client import AsyncEngineRPCClient
43
from vllm.entrypoints.openai.rpc.server import run_rpc_server
44
+# yapf: enable
45
from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
46
from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
47
from vllm.entrypoints.openai.serving_embedding import OpenAIServingEmbedding
@@ -343,6 +345,40 @@ if envs.VLLM_TORCH_PROFILER_DIR:
345
return Response(status_code=200)
346
347
348
+if envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING:
349
+ logger.warning(
350
+ "Lora dynamic loading & unloading is enabled in the API server. "
351
+ "This should ONLY be used for local development!")
352
+
353
+ @router.post("/v1/load_lora_adapter")
354
+ async def load_lora_adapter(request: LoadLoraAdapterRequest):
355
+ response = await openai_serving_chat.load_lora_adapter(request)
356
+ if isinstance(response, ErrorResponse):
357
+ return JSONResponse(content=response.model_dump(),
358
+ status_code=response.code)
359
+
360
+ response = await openai_serving_completion.load_lora_adapter(request)
361
+ if isinstance(response, ErrorResponse):
362
+ return JSONResponse(content=response.model_dump(),
363
+ status_code=response.code)
364
+
365
+ return Response(status_code=200, content=response)
366
+
367
+ @router.post("/v1/unload_lora_adapter")
368
+ async def unload_lora_adapter(request: UnloadLoraAdapterRequest):
369
+ response = await openai_serving_chat.unload_lora_adapter(request)
370
+ if isinstance(response, ErrorResponse):
371
+ return JSONResponse(content=response.model_dump(),
372
+ status_code=response.code)
373
+
374
+ response = await openai_serving_completion.unload_lora_adapter(request)
375
+ if isinstance(response, ErrorResponse):
376
+ return JSONResponse(content=response.model_dump(),
377
+ status_code=response.code)
378
+
379
+ return Response(status_code=200, content=response)
380
+
381
+
382
def build_app(args: Namespace) -> FastAPI:
383
app = FastAPI(lifespan=lifespan)
384
app.include_router(router)
vllm/entrypoints/openai/protocol.py
+10
@@ -878,3 +878,13 @@ class DetokenizeRequest(OpenAIBaseModel):
878
879
class DetokenizeResponse(OpenAIBaseModel):
880
prompt: str
881
+
882
+
883
+class LoadLoraAdapterRequest(BaseModel):
884
+ lora_name: str
885
+ lora_path: str
886
+
887
+
888
+class UnloadLoraAdapterRequest(BaseModel):
889
+ lora_name: str
890
+ lora_int_id: Optional[int] = Field(default=None)
vllm/entrypoints/openai/serving_engine.py
+78
-1
@@ -16,11 +16,13 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
16
CompletionRequest,
17
DetokenizeRequest,
18
EmbeddingRequest, ErrorResponse,
19
+ LoadLoraAdapterRequest,
20
ModelCard, ModelList,
21
ModelPermission,
22
TokenizeChatRequest,
23
TokenizeCompletionRequest,
23
- TokenizeRequest)
24
+ TokenizeRequest,
25
+ UnloadLoraAdapterRequest)
26
# yapf: enable
27
from vllm.inputs.parse import parse_and_batch_prompt
28
from vllm.logger import init_logger
@@ -32,6 +34,7 @@ from vllm.prompt_adapter.request import PromptAdapterRequest
34
from vllm.sampling_params import LogitsProcessor, SamplingParams
35
from vllm.sequence import Logprob
36
from vllm.transformers_utils.tokenizer import AnyTokenizer
37
+from vllm.utils import AtomicCounter
38
39
logger = init_logger(__name__)
40
@@ -78,6 +81,7 @@ class OpenAIServing:
81
82
self.served_model_names = served_model_names
83
84
+ self.lora_id_counter = AtomicCounter(0)
85
self.lora_requests = []
86
if lora_modules is not None:
87
self.lora_requests = [
@@ -403,3 +407,76 @@ class OpenAIServing:
407
if logprob.decoded_token is not None:
408
return logprob.decoded_token
409
return tokenizer.decode(token_id)
410
+
411
+ async def _check_load_lora_adapter_request(
412
+ self, request: LoadLoraAdapterRequest) -> Optional[ErrorResponse]:
413
+ # Check if both 'lora_name' and 'lora_path' are provided
414
+ if not request.lora_name or not request.lora_path:
415
+ return self.create_error_response(
416
+ message="Both 'lora_name' and 'lora_path' must be provided.",
417
+ err_type="InvalidUserInput",
418
+ status_code=HTTPStatus.BAD_REQUEST)
419
+
420
+ # Check if the lora adapter with the given name already exists
421
+ if any(lora_request.lora_name == request.lora_name
422
+ for lora_request in self.lora_requests):
423
+ return self.create_error_response(
424
+ message=
425
+ f"The lora adapter '{request.lora_name}' has already been"
426
+ "loaded.",
427
+ err_type="InvalidUserInput",
428
+ status_code=HTTPStatus.BAD_REQUEST)
429
+
430
+ return None
431
+
432
+ async def _check_unload_lora_adapter_request(
433
+ self,
434
+ request: UnloadLoraAdapterRequest) -> Optional[ErrorResponse]:
435
+ # Check if either 'lora_name' or 'lora_int_id' is provided
436
+ if not request.lora_name and not request.lora_int_id:
437
+ return self.create_error_response(
438
+ message=
439
+ "either 'lora_name' and 'lora_int_id' needs to be provided.",
440
+ err_type="InvalidUserInput",
441
+ status_code=HTTPStatus.BAD_REQUEST)
442
+
443
+ # Check if the lora adapter with the given name exists
444
+ if not any(lora_request.lora_name == request.lora_name
445
+ for lora_request in self.lora_requests):
446
+ return self.create_error_response(
447
+ message=
448
+ f"The lora adapter '{request.lora_name}' cannot be found.",
449
+ err_type="InvalidUserInput",
450
+ status_code=HTTPStatus.BAD_REQUEST)
451
+
452
+ return None
453
+
454
+ async def load_lora_adapter(
455
+ self,
456
+ request: LoadLoraAdapterRequest) -> Union[ErrorResponse, str]:
457
+ error_check_ret = await self._check_load_lora_adapter_request(request)
458
+ if error_check_ret is not None:
459
+ return error_check_ret
460
+
461
+ lora_name, lora_path = request.lora_name, request.lora_path
462
+ unique_id = self.lora_id_counter.inc(1)
463
+ self.lora_requests.append(
464
+ LoRARequest(lora_name=lora_name,
465
+ lora_int_id=unique_id,
466
+ lora_path=lora_path))
467
+ return f"Success: LoRA adapter '{lora_name}' added successfully."
468
+
469
+ async def unload_lora_adapter(
470
+ self,
471
+ request: UnloadLoraAdapterRequest) -> Union[ErrorResponse, str]:
472
+ error_check_ret = await self._check_unload_lora_adapter_request(request
473
+ )
474
+ if error_check_ret is not None:
475
+ return error_check_ret
476
+
477
+ lora_name = request.lora_name
478
+ self.lora_requests = [
479
+ lora_request for lora_request in self.lora_requests
480
+ if lora_request.lora_name != lora_name
481
+ ]
482
+ return f"Success: LoRA adapter '{lora_name}' removed successfully."
vllm/envs.py
+7
@@ -61,6 +61,7 @@ if TYPE_CHECKING:
61
VLLM_ALLOW_ENGINE_USE_RAY: bool = False
62
VLLM_PLUGINS: Optional[List[str]] = None
63
VLLM_TORCH_PROFILER_DIR: Optional[str] = None
64
+ VLLM_ALLOW_RUNTIME_LORA_UPDATING: bool = False
65
66
67
def get_default_cache_root():
@@ -409,6 +410,12 @@ environment_variables: Dict[str, Callable[[], Any]] = {
410
# If set, vLLM will use Triton implementations of AWQ.
411
"VLLM_USE_TRITON_AWQ":
412
lambda: bool(int(os.getenv("VLLM_USE_TRITON_AWQ", "0"))),
413
+
414
+ # If set, allow loading or unloading lora adapters in runtime,
415
+ "VLLM_ALLOW_RUNTIME_LORA_UPDATING":
416
+ lambda:
417
+ (os.environ.get("VLLM_ALLOW_RUNTIME_LORA_UPDATING", "0").strip().lower() in
418
+ ("1", "true")),
419
}
420
421
# end-env-vars-definition
vllm/lora/request.py
+18
-1
@@ -28,7 +28,6 @@ class LoRARequest(
28
lora_path: str = ""
29
lora_local_path: Optional[str] = msgspec.field(default=None)
30
long_lora_max_len: Optional[int] = None
31
- __hash__ = AdapterRequest.__hash__
31
32
def __post_init__(self):
33
if 'lora_local_path' in self.__struct_fields__:
@@ -75,3 +74,21 @@ class LoRARequest(
74
DeprecationWarning,
75
stacklevel=2)
76
self.lora_path = value
77
+
78
+ def __eq__(self, value: object) -> bool:
79
+ """
80
+ Overrides the equality method to compare LoRARequest
81
+ instances based on lora_name. This allows for identification
82
+ and comparison lora adapter across engines.
83
+ """
84
+ return isinstance(value,
85
+ self.__class__) and self.lora_name == value.lora_name
86
+
87
+ def __hash__(self) -> int:
88
+ """
89
+ Overrides the hash method to hash LoRARequest instances
90
+ based on lora_name. This ensures that LoRARequest instances
91
+ can be used in hash-based collections such as sets and dictionaries,
92
+ identified by their names across engines.
93
+ """
94
+ return hash(self.lora_name)
vllm/utils.py
+25
@@ -1224,3 +1224,28 @@ async def _run_task_with_lock(task: Callable, lock: asyncio.Lock, *args,
1224
def supports_dynamo() -> bool:
1225
base_torch_version = Version(Version(torch.__version__).base_version)
1226
return base_torch_version >= Version("2.4.0")
1227
+
1228
+
1229
+class AtomicCounter:
1230
+ """An atomic, thread-safe counter"""
1231
+
1232
+ def __init__(self, initial=0):
1233
+ """Initialize a new atomic counter to given initial value"""
1234
+ self._value = initial
1235
+ self._lock = threading.Lock()
1236
+
1237
+ def inc(self, num=1):
1238
+ """Atomically increment the counter by num and return the new value"""
1239
+ with self._lock:
1240
+ self._value += num
1241
+ return self._value
1242
+
1243
+ def dec(self, num=1):
1244
+ """Atomically decrement the counter by num and return the new value"""
1245
+ with self._lock:
1246
+ self._value -= num
1247
+ return self._value
1248
+
1249
+ @property
1250
+ def value(self):
1251
+ return self._value