@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / db3bf7c991
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Core] Support load and unload LoRA in api server (#6566)

Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>

Jiaxin Shan committed Sep 5, 2024 at 18:10 UTC db3bf7c991cd1a0297d1a8ba501e59cfa226c337
10 files changed +336 -6
docs/requirements-docs.txt
-1
@@ -11,6 +11,5 @@ pydantic >= 2.8
11 torch
12 py-cpuinfo
13 transformers
14 -openai # Required by docs/source/serving/openai_compatible_server.md's vllm.entrypoints.openai.cli_args
14 mistral_common >= 1.3.4
15 openai # Required by docs/source/serving/openai_compatible_server.md's vllm.entrypoints.openai.cli_args
\ No newline at end of file
docs/source/models/lora.rst
+52
@@ -107,3 +107,55 @@ The following is an example request
107 "max_tokens": 7,
108 "temperature": 0
109 }' | jq
110 +
111 +
112 +Dynamically serving LoRA Adapters
113 +---------------------------------
114 +
115 +In addition to serving LoRA adapters at server startup, the vLLM server now supports dynamically loading and unloading
116 +LoRA adapters at runtime through dedicated API endpoints. This feature can be particularly useful when the flexibility
117 +to change models on-the-fly is needed.
118 +
119 +Note: Enabling this feature in production environments is risky as user may participate model adapter management.
120 +
121 +To enable dynamic LoRA loading and unloading, ensure that the environment variable `VLLM_ALLOW_RUNTIME_LORA_UPDATING`
122 +is set to `True`. When this option is enabled, the API server will log a warning to indicate that dynamic loading is active.
123 +
124 +.. code-block:: bash
125 +
126 + export VLLM_ALLOW_RUNTIME_LORA_UPDATING=True
127 +
128 +
129 +Loading a LoRA Adapter:
130 +
131 +To dynamically load a LoRA adapter, send a POST request to the `/v1/load_lora_adapter` endpoint with the necessary
132 +details of the adapter to be loaded. The request payload should include the name and path to the LoRA adapter.
133 +
134 +Example request to load a LoRA adapter:
135 +
136 +.. code-block:: bash
137 +
138 + curl -X POST http://localhost:8000/v1/load_lora_adapter \
139 + -H "Content-Type: application/json" \
140 + -d '{
141 + "lora_name": "sql_adapter",
142 + "lora_path": "/path/to/sql-lora-adapter"
143 + }'
144 +
145 +Upon a successful request, the API will respond with a 200 OK status code. If an error occurs, such as if the adapter
146 +cannot be found or loaded, an appropriate error message will be returned.
147 +
148 +Unloading a LoRA Adapter:
149 +
150 +To unload a LoRA adapter that has been previously loaded, send a POST request to the `/v1/unload_lora_adapter` endpoint
151 +with the name or ID of the adapter to be unloaded.
152 +
153 +Example request to unload a LoRA adapter:
154 +
155 +.. code-block:: bash
156 +
157 + curl -X POST http://localhost:8000/v1/unload_lora_adapter \
158 + -H "Content-Type: application/json" \
159 + -d '{
160 + "lora_name": "sql_adapter"
161 + }'
tests/entrypoints/llm/test_generate_multiple_loras.py
+1 -1
@@ -50,7 +50,7 @@ def zephyr_lora_files():
50 @pytest.mark.skip_global_cleanup
51 def test_multiple_lora_requests(llm: LLM, zephyr_lora_files):
52 lora_request = [
53 - LoRARequest(LORA_NAME, idx + 1, zephyr_lora_files)
53 + LoRARequest(LORA_NAME + str(idx), idx + 1, zephyr_lora_files)
54 for idx in range(len(PROMPTS))
55 ]
56 # Multiple SamplingParams should be matched with each prompt
tests/entrypoints/openai/test_serving_engine.py new
+107
@@ -0,0 +1,107 @@
1 +from http import HTTPStatus
2 +from unittest.mock import MagicMock
3 +
4 +import pytest
5 +
6 +from vllm.config import ModelConfig
7 +from vllm.engine.protocol import AsyncEngineClient
8 +from vllm.entrypoints.openai.protocol import (ErrorResponse,
9 + LoadLoraAdapterRequest,
10 + UnloadLoraAdapterRequest)
11 +from vllm.entrypoints.openai.serving_engine import OpenAIServing
12 +
13 +MODEL_NAME = "meta-llama/Llama-2-7b"
14 +LORA_LOADING_SUCCESS_MESSAGE = (
15 + "Success: LoRA adapter '{lora_name}' added successfully.")
16 +LORA_UNLOADING_SUCCESS_MESSAGE = (
17 + "Success: LoRA adapter '{lora_name}' removed successfully.")
18 +
19 +
20 +async def _async_serving_engine_init():
21 + mock_engine_client = MagicMock(spec=AsyncEngineClient)
22 + mock_model_config = MagicMock(spec=ModelConfig)
23 + # Set the max_model_len attribute to avoid missing attribute
24 + mock_model_config.max_model_len = 2048
25 +
26 + serving_engine = OpenAIServing(mock_engine_client,
27 + mock_model_config,
28 + served_model_names=[MODEL_NAME],
29 + lora_modules=None,
30 + prompt_adapters=None,
31 + request_logger=None)
32 + return serving_engine
33 +
34 +
35 +@pytest.mark.asyncio
36 +async def test_load_lora_adapter_success():
37 + serving_engine = await _async_serving_engine_init()
38 + request = LoadLoraAdapterRequest(lora_name="adapter",
39 + lora_path="/path/to/adapter2")
40 + response = await serving_engine.load_lora_adapter(request)
41 + assert response == LORA_LOADING_SUCCESS_MESSAGE.format(lora_name='adapter')
42 + assert len(serving_engine.lora_requests) == 1
43 + assert serving_engine.lora_requests[0].lora_name == "adapter"
44 +
45 +
46 +@pytest.mark.asyncio
47 +async def test_load_lora_adapter_missing_fields():
48 + serving_engine = await _async_serving_engine_init()
49 + request = LoadLoraAdapterRequest(lora_name="", lora_path="")
50 + response = await serving_engine.load_lora_adapter(request)
51 + assert isinstance(response, ErrorResponse)
52 + assert response.type == "InvalidUserInput"
53 + assert response.code == HTTPStatus.BAD_REQUEST
54 +
55 +
56 +@pytest.mark.asyncio
57 +async def test_load_lora_adapter_duplicate():
58 + serving_engine = await _async_serving_engine_init()
59 + request = LoadLoraAdapterRequest(lora_name="adapter1",
60 + lora_path="/path/to/adapter1")
61 + response = await serving_engine.load_lora_adapter(request)
62 + assert response == LORA_LOADING_SUCCESS_MESSAGE.format(
63 + lora_name='adapter1')
64 + assert len(serving_engine.lora_requests) == 1
65 +
66 + request = LoadLoraAdapterRequest(lora_name="adapter1",
67 + lora_path="/path/to/adapter1")
68 + response = await serving_engine.load_lora_adapter(request)
69 + assert isinstance(response, ErrorResponse)
70 + assert response.type == "InvalidUserInput"
71 + assert response.code == HTTPStatus.BAD_REQUEST
72 + assert len(serving_engine.lora_requests) == 1
73 +
74 +
75 +@pytest.mark.asyncio
76 +async def test_unload_lora_adapter_success():
77 + serving_engine = await _async_serving_engine_init()
78 + request = LoadLoraAdapterRequest(lora_name="adapter1",
79 + lora_path="/path/to/adapter1")
80 + response = await serving_engine.load_lora_adapter(request)
81 + assert len(serving_engine.lora_requests) == 1
82 +
83 + request = UnloadLoraAdapterRequest(lora_name="adapter1")
84 + response = await serving_engine.unload_lora_adapter(request)
85 + assert response == LORA_UNLOADING_SUCCESS_MESSAGE.format(
86 + lora_name='adapter1')
87 + assert len(serving_engine.lora_requests) == 0
88 +
89 +
90 +@pytest.mark.asyncio
91 +async def test_unload_lora_adapter_missing_fields():
92 + serving_engine = await _async_serving_engine_init()
93 + request = UnloadLoraAdapterRequest(lora_name="", lora_int_id=None)
94 + response = await serving_engine.unload_lora_adapter(request)
95 + assert isinstance(response, ErrorResponse)
96 + assert response.type == "InvalidUserInput"
97 + assert response.code == HTTPStatus.BAD_REQUEST
98 +
99 +
100 +@pytest.mark.asyncio
101 +async def test_unload_lora_adapter_not_found():
102 + serving_engine = await _async_serving_engine_init()
103 + request = UnloadLoraAdapterRequest(lora_name="nonexistent_adapter")
104 + response = await serving_engine.unload_lora_adapter(request)
105 + assert isinstance(response, ErrorResponse)
106 + assert response.type == "InvalidUserInput"
107 + assert response.code == HTTPStatus.BAD_REQUEST
vllm/entrypoints/openai/api_server.py
+38 -2
@@ -35,11 +35,13 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
35 DetokenizeResponse,
36 EmbeddingRequest,
37 EmbeddingResponse, ErrorResponse,
38 + LoadLoraAdapterRequest,
39 TokenizeRequest,
39 - TokenizeResponse)
40 -# yapf: enable
40 + TokenizeResponse,
41 + UnloadLoraAdapterRequest)
42 from vllm.entrypoints.openai.rpc.client import AsyncEngineRPCClient
43 from vllm.entrypoints.openai.rpc.server import run_rpc_server
44 +# yapf: enable
45 from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
46 from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
47 from vllm.entrypoints.openai.serving_embedding import OpenAIServingEmbedding
@@ -343,6 +345,40 @@ if envs.VLLM_TORCH_PROFILER_DIR:
345 return Response(status_code=200)
346
347
348 +if envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING:
349 + logger.warning(
350 + "Lora dynamic loading & unloading is enabled in the API server. "
351 + "This should ONLY be used for local development!")
352 +
353 + @router.post("/v1/load_lora_adapter")
354 + async def load_lora_adapter(request: LoadLoraAdapterRequest):
355 + response = await openai_serving_chat.load_lora_adapter(request)
356 + if isinstance(response, ErrorResponse):
357 + return JSONResponse(content=response.model_dump(),
358 + status_code=response.code)
359 +
360 + response = await openai_serving_completion.load_lora_adapter(request)
361 + if isinstance(response, ErrorResponse):
362 + return JSONResponse(content=response.model_dump(),
363 + status_code=response.code)
364 +
365 + return Response(status_code=200, content=response)
366 +
367 + @router.post("/v1/unload_lora_adapter")
368 + async def unload_lora_adapter(request: UnloadLoraAdapterRequest):
369 + response = await openai_serving_chat.unload_lora_adapter(request)
370 + if isinstance(response, ErrorResponse):
371 + return JSONResponse(content=response.model_dump(),
372 + status_code=response.code)
373 +
374 + response = await openai_serving_completion.unload_lora_adapter(request)
375 + if isinstance(response, ErrorResponse):
376 + return JSONResponse(content=response.model_dump(),
377 + status_code=response.code)
378 +
379 + return Response(status_code=200, content=response)
380 +
381 +
382 def build_app(args: Namespace) -> FastAPI:
383 app = FastAPI(lifespan=lifespan)
384 app.include_router(router)
vllm/entrypoints/openai/protocol.py
+10
@@ -878,3 +878,13 @@ class DetokenizeRequest(OpenAIBaseModel):
878
879 class DetokenizeResponse(OpenAIBaseModel):
880 prompt: str
881 +
882 +
883 +class LoadLoraAdapterRequest(BaseModel):
884 + lora_name: str
885 + lora_path: str
886 +
887 +
888 +class UnloadLoraAdapterRequest(BaseModel):
889 + lora_name: str
890 + lora_int_id: Optional[int] = Field(default=None)
vllm/entrypoints/openai/serving_engine.py
+78 -1
@@ -16,11 +16,13 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
16 CompletionRequest,
17 DetokenizeRequest,
18 EmbeddingRequest, ErrorResponse,
19 + LoadLoraAdapterRequest,
20 ModelCard, ModelList,
21 ModelPermission,
22 TokenizeChatRequest,
23 TokenizeCompletionRequest,
23 - TokenizeRequest)
24 + TokenizeRequest,
25 + UnloadLoraAdapterRequest)
26 # yapf: enable
27 from vllm.inputs.parse import parse_and_batch_prompt
28 from vllm.logger import init_logger
@@ -32,6 +34,7 @@ from vllm.prompt_adapter.request import PromptAdapterRequest
34 from vllm.sampling_params import LogitsProcessor, SamplingParams
35 from vllm.sequence import Logprob
36 from vllm.transformers_utils.tokenizer import AnyTokenizer
37 +from vllm.utils import AtomicCounter
38
39 logger = init_logger(__name__)
40
@@ -78,6 +81,7 @@ class OpenAIServing:
81
82 self.served_model_names = served_model_names
83
84 + self.lora_id_counter = AtomicCounter(0)
85 self.lora_requests = []
86 if lora_modules is not None:
87 self.lora_requests = [
@@ -403,3 +407,76 @@ class OpenAIServing:
407 if logprob.decoded_token is not None:
408 return logprob.decoded_token
409 return tokenizer.decode(token_id)
410 +
411 + async def _check_load_lora_adapter_request(
412 + self, request: LoadLoraAdapterRequest) -> Optional[ErrorResponse]:
413 + # Check if both 'lora_name' and 'lora_path' are provided
414 + if not request.lora_name or not request.lora_path:
415 + return self.create_error_response(
416 + message="Both 'lora_name' and 'lora_path' must be provided.",
417 + err_type="InvalidUserInput",
418 + status_code=HTTPStatus.BAD_REQUEST)
419 +
420 + # Check if the lora adapter with the given name already exists
421 + if any(lora_request.lora_name == request.lora_name
422 + for lora_request in self.lora_requests):
423 + return self.create_error_response(
424 + message=
425 + f"The lora adapter '{request.lora_name}' has already been"
426 + "loaded.",
427 + err_type="InvalidUserInput",
428 + status_code=HTTPStatus.BAD_REQUEST)
429 +
430 + return None
431 +
432 + async def _check_unload_lora_adapter_request(
433 + self,
434 + request: UnloadLoraAdapterRequest) -> Optional[ErrorResponse]:
435 + # Check if either 'lora_name' or 'lora_int_id' is provided
436 + if not request.lora_name and not request.lora_int_id:
437 + return self.create_error_response(
438 + message=
439 + "either 'lora_name' and 'lora_int_id' needs to be provided.",
440 + err_type="InvalidUserInput",
441 + status_code=HTTPStatus.BAD_REQUEST)
442 +
443 + # Check if the lora adapter with the given name exists
444 + if not any(lora_request.lora_name == request.lora_name
445 + for lora_request in self.lora_requests):
446 + return self.create_error_response(
447 + message=
448 + f"The lora adapter '{request.lora_name}' cannot be found.",
449 + err_type="InvalidUserInput",
450 + status_code=HTTPStatus.BAD_REQUEST)
451 +
452 + return None
453 +
454 + async def load_lora_adapter(
455 + self,
456 + request: LoadLoraAdapterRequest) -> Union[ErrorResponse, str]:
457 + error_check_ret = await self._check_load_lora_adapter_request(request)
458 + if error_check_ret is not None:
459 + return error_check_ret
460 +
461 + lora_name, lora_path = request.lora_name, request.lora_path
462 + unique_id = self.lora_id_counter.inc(1)
463 + self.lora_requests.append(
464 + LoRARequest(lora_name=lora_name,
465 + lora_int_id=unique_id,
466 + lora_path=lora_path))
467 + return f"Success: LoRA adapter '{lora_name}' added successfully."
468 +
469 + async def unload_lora_adapter(
470 + self,
471 + request: UnloadLoraAdapterRequest) -> Union[ErrorResponse, str]:
472 + error_check_ret = await self._check_unload_lora_adapter_request(request
473 + )
474 + if error_check_ret is not None:
475 + return error_check_ret
476 +
477 + lora_name = request.lora_name
478 + self.lora_requests = [
479 + lora_request for lora_request in self.lora_requests
480 + if lora_request.lora_name != lora_name
481 + ]
482 + return f"Success: LoRA adapter '{lora_name}' removed successfully."
vllm/envs.py
+7
@@ -61,6 +61,7 @@ if TYPE_CHECKING:
61 VLLM_ALLOW_ENGINE_USE_RAY: bool = False
62 VLLM_PLUGINS: Optional[List[str]] = None
63 VLLM_TORCH_PROFILER_DIR: Optional[str] = None
64 + VLLM_ALLOW_RUNTIME_LORA_UPDATING: bool = False
65
66
67 def get_default_cache_root():
@@ -409,6 +410,12 @@ environment_variables: Dict[str, Callable[[], Any]] = {
410 # If set, vLLM will use Triton implementations of AWQ.
411 "VLLM_USE_TRITON_AWQ":
412 lambda: bool(int(os.getenv("VLLM_USE_TRITON_AWQ", "0"))),
413 +
414 + # If set, allow loading or unloading lora adapters in runtime,
415 + "VLLM_ALLOW_RUNTIME_LORA_UPDATING":
416 + lambda:
417 + (os.environ.get("VLLM_ALLOW_RUNTIME_LORA_UPDATING", "0").strip().lower() in
418 + ("1", "true")),
419 }
420
421 # end-env-vars-definition
vllm/lora/request.py
+18 -1
@@ -28,7 +28,6 @@ class LoRARequest(
28 lora_path: str = ""
29 lora_local_path: Optional[str] = msgspec.field(default=None)
30 long_lora_max_len: Optional[int] = None
31 - __hash__ = AdapterRequest.__hash__
31
32 def __post_init__(self):
33 if 'lora_local_path' in self.__struct_fields__:
@@ -75,3 +74,21 @@ class LoRARequest(
74 DeprecationWarning,
75 stacklevel=2)
76 self.lora_path = value
77 +
78 + def __eq__(self, value: object) -> bool:
79 + """
80 + Overrides the equality method to compare LoRARequest
81 + instances based on lora_name. This allows for identification
82 + and comparison lora adapter across engines.
83 + """
84 + return isinstance(value,
85 + self.__class__) and self.lora_name == value.lora_name
86 +
87 + def __hash__(self) -> int:
88 + """
89 + Overrides the hash method to hash LoRARequest instances
90 + based on lora_name. This ensures that LoRARequest instances
91 + can be used in hash-based collections such as sets and dictionaries,
92 + identified by their names across engines.
93 + """
94 + return hash(self.lora_name)
vllm/utils.py
+25
@@ -1224,3 +1224,28 @@ async def _run_task_with_lock(task: Callable, lock: asyncio.Lock, *args,
1224 def supports_dynamo() -> bool:
1225 base_torch_version = Version(Version(torch.__version__).base_version)
1226 return base_torch_version >= Version("2.4.0")
1227 +
1228 +
1229 +class AtomicCounter:
1230 + """An atomic, thread-safe counter"""
1231 +
1232 + def __init__(self, initial=0):
1233 + """Initialize a new atomic counter to given initial value"""
1234 + self._value = initial
1235 + self._lock = threading.Lock()
1236 +
1237 + def inc(self, num=1):
1238 + """Atomically increment the counter by num and return the new value"""
1239 + with self._lock:
1240 + self._value += num
1241 + return self._value
1242 +
1243 + def dec(self, num=1):
1244 + """Atomically decrement the counter by num and return the new value"""
1245 + with self._lock:
1246 + self._value -= num
1247 + return self._value
1248 +
1249 + @property
1250 + def value(self):
1251 + return self._value