@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cbc40128eb
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[V1] LoRA - Enable Serving Usecase (#12883)

Signed-off-by: Varun Sundar Rabindranath <varun@neuralmagic.com> Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com>

Varun Sundar Rabindranath committed Feb 14, 2025 at 11:51 UTC cbc40128eb16ae0045f5aa6d6ee2ff2dda803e23
7 files changed +210 -7
tests/lora/test_add_lora.py new
+165
@@ -0,0 +1,165 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +import asyncio
3 +import time
4 +from pathlib import Path
5 +from typing import List
6 +
7 +import pytest
8 +from huggingface_hub import snapshot_download
9 +
10 +from vllm.engine.arg_utils import AsyncEngineArgs
11 +from vllm.inputs import TextPrompt
12 +from vllm.lora.request import LoRARequest
13 +from vllm.sampling_params import SamplingParams
14 +from vllm.utils import merge_async_iterators
15 +
16 +MODEL_PATH = "meta-llama/Llama-2-7b-hf"
17 +LORA_MODULE_DOWNLOAD_PATH = None # Populated by download_and_prepare_lora_module() #noqa
18 +LORA_RANK = 8
19 +DEFAULT_MAX_LORAS = 16 * 3
20 +
21 +
22 +def download_and_prepare_lora_module():
23 + """
24 + Request submission is expensive when the LoRA adapters have their own
25 + tokenizers. This is because, for each request with a new LoRA adapter ID,
26 + the front-end loads the tokenizer from disk.
27 +
28 + In this test, as we are comparing request processing times, we want to
29 + minimize any extra activity. To this effect, we download the LoRA
30 + adapter and remove all the tokenizer files, so the engine will default
31 + to the base model tokenizer.
32 + """
33 + global LORA_MODULE_DOWNLOAD_PATH
34 +
35 + LORA_MODULE_HF_PATH = "yard1/llama-2-7b-sql-lora-test"
36 + LORA_MODULE_DOWNLOAD_PATH = snapshot_download(repo_id=LORA_MODULE_HF_PATH)
37 +
38 + tokenizer_files = [
39 + 'added_tokens.json', 'tokenizer_config.json', 'tokenizer.json',
40 + 'tokenizer.model'
41 + ]
42 + for tokenizer_file in tokenizer_files:
43 + del_path = Path(LORA_MODULE_DOWNLOAD_PATH) / tokenizer_file
44 + del_path.unlink()
45 +
46 +
47 +@pytest.fixture(autouse=True)
48 +def v1(run_with_both_engines_lora):
49 + # Simple autouse wrapper to run both engines for each test
50 + # This can be promoted up to conftest.py to run for every
51 + # test in a package
52 + pass
53 +
54 +
55 +def get_lora_requests() -> List[LoRARequest]:
56 + lora_requests: List[LoRARequest] = [
57 + LoRARequest(lora_name=f"{i}",
58 + lora_int_id=i,
59 + lora_path=LORA_MODULE_DOWNLOAD_PATH)
60 + for i in range(1, DEFAULT_MAX_LORAS + 1)
61 + ]
62 + return lora_requests
63 +
64 +
65 +async def requests_processing_time(llm,
66 + lora_requests: List[LoRARequest]) -> float:
67 +
68 + sampling_params = SamplingParams(n=1,
69 + temperature=0.0,
70 + top_p=1.0,
71 + ignore_eos=True,
72 + max_tokens=1)
73 +
74 + generators = []
75 + start = time.perf_counter()
76 +
77 + for lora_request in lora_requests:
78 + lora_int_id = lora_request.lora_int_id
79 + generator = llm.generate(
80 + prompt=TextPrompt(prompt=f"hello {lora_int_id}",
81 + multi_modal_data=None), # type: ignore
82 + sampling_params=sampling_params,
83 + lora_request=lora_request,
84 + request_id=f"test{lora_int_id}")
85 + generators.append(generator)
86 +
87 + all_gens = merge_async_iterators(*generators)
88 + async for i, res in all_gens:
89 + pass
90 +
91 + end = time.perf_counter()
92 + return end - start
93 +
94 +
95 +@pytest.mark.asyncio
96 +async def test_add_lora():
97 + """
98 + The add_lora function is used to pre-load some LoRA adapters into the
99 + engine in anticipation of future requests using these adapters. To test
100 + this functionality, we use the async engine to process some requests - We
101 + do it twice, once with add_lora() pre-loading and once without.
102 +
103 + We measure the request processing time in both cases and expect the time
104 + to be lesser in the case with add_lora() calls.
105 + """
106 +
107 + download_and_prepare_lora_module()
108 +
109 + lora_requests: List[LoRARequest] = get_lora_requests()
110 +
111 + max_loras = len(set([lr.lora_int_id for lr in lora_requests]))
112 + # Create engine in eager-mode. Due to high max_loras, the CI can
113 + # OOM during cuda-graph capture.
114 + engine_args = AsyncEngineArgs(
115 + model=MODEL_PATH,
116 + enable_lora=True,
117 + max_loras=max_loras,
118 + max_lora_rank=LORA_RANK,
119 + max_model_len=128,
120 + gpu_memory_utilization=0.8, #avoid OOM
121 + enforce_eager=True)
122 +
123 + # The run_with_both_engines_lora fixture sets up the `VLLM_USE_V1`
124 + # environment variable. reload vllm.enging.async_llm_engine as
125 + # vllm.engine.async_llm_engine.AsyncLLMEgnine changes depending on the
126 + # env var.
127 + import importlib
128 +
129 + import vllm.engine.async_llm_engine
130 + importlib.reload(vllm.engine.async_llm_engine)
131 + from vllm.entrypoints.openai.api_server import (
132 + build_async_engine_client_from_engine_args)
133 +
134 + # split lora_requests into 3 parts
135 + part_size = len(lora_requests) // 3
136 + dummy_run_requests = lora_requests[:part_size]
137 + warmup_run_requests = lora_requests[part_size:part_size * 2]
138 + cold_run_requests = lora_requests[part_size * 2:]
139 +
140 + async with build_async_engine_client_from_engine_args(engine_args) as llm:
141 +
142 + # Dummy run - So any 1-time functionality like triton kernel compilation
143 + # is complete here.
144 + await requests_processing_time(llm, dummy_run_requests)
145 +
146 + # Run with warmup
147 + for lr in warmup_run_requests:
148 + await llm.add_lora(lr)
149 + # Wait for the add_lora function to complete on the server side.
150 + await asyncio.sleep(30)
151 + time_with_add_lora = await requests_processing_time(
152 + llm, warmup_run_requests)
153 +
154 + # Run without any warmup
155 + time_cold_start = await requests_processing_time(
156 + llm, cold_run_requests)
157 +
158 + print(f"time hot-start {time_with_add_lora} vs "
159 + f"time cold-start {time_cold_start} ")
160 +
161 + assert time_with_add_lora < time_cold_start, (
162 + f"time_with_add_lora={time_with_add_lora}, "
163 + f"time_cold_start={time_cold_start}"
164 + "The engine request processing time with LoRA pre-loading "
165 + "must be less than the version that does on-demand LoRA loading.")
vllm/v1/engine/__init__.py
+1
@@ -134,3 +134,4 @@ class EngineCoreRequestType(enum.Enum):
134 ABORT = b'\x01'
135 PROFILE = b'\x02'
136 RESET_PREFIX_CACHE = b'\x03'
137 + ADD_LORA = b'\x04'
vllm/v1/engine/async_llm.py
+4 -4
@@ -361,6 +361,10 @@ class AsyncLLM(EngineClient):
361 async def reset_prefix_cache(self) -> None:
362 await self.engine_core.reset_prefix_cache_async()
363
364 + async def add_lora(self, lora_request: LoRARequest) -> None:
365 + """Load a new LoRA adapter into the engine for future requests."""
366 + await self.engine_core.add_lora_async(lora_request)
367 +
368 @property
369 def is_running(self) -> bool:
370 return True
@@ -376,7 +380,3 @@ class AsyncLLM(EngineClient):
380 @property
381 def dead_error(self) -> BaseException:
382 return Exception() # TODO: implement
379 -
380 - async def add_lora(self, lora_request: LoRARequest) -> None:
381 - """Load a new LoRA adapter into the engine for future requests."""
382 - raise NotImplementedError("LoRA not yet supported in V1")
vllm/v1/engine/core.py
+15 -3
@@ -13,6 +13,7 @@ import zmq.asyncio
13
14 from vllm.config import VllmConfig
15 from vllm.logger import init_logger
16 +from vllm.lora.request import LoRARequest
17 from vllm.transformers_utils.config import (
18 maybe_register_config_serialize_by_value)
19 from vllm.utils import get_exception_traceback, zmq_socket_ctx
@@ -146,6 +147,9 @@ class EngineCore:
147 def reset_prefix_cache(self):
148 self.scheduler.reset_prefix_cache()
149
150 + def add_lora(self, lora_request: LoRARequest) -> None:
151 + self.model_executor.add_lora(lora_request)
152 +
153
154 class EngineCoreProc(EngineCore):
155 """ZMQ-wrapper for running EngineCore in background process."""
@@ -262,12 +266,15 @@ class EngineCoreProc(EngineCore):
266 self.reset_prefix_cache()
267 elif request_type == EngineCoreRequestType.PROFILE:
268 self.model_executor.profile(request)
269 + elif request_type == EngineCoreRequestType.ADD_LORA:
270 + self.model_executor.add_lora(request)
271
272 def process_input_socket(self, input_path: str):
273 """Input socket IO thread."""
274
275 # Msgpack serialization decoding.
276 add_request_decoder = MsgpackDecoder(EngineCoreRequest)
277 + add_lora_decoder = MsgpackDecoder(LoRARequest)
278 generic_decoder = MsgpackDecoder()
279
280 with zmq_socket_ctx(input_path, zmq.constants.PULL) as socket:
@@ -277,9 +284,14 @@ class EngineCoreProc(EngineCore):
284 request_type = EngineCoreRequestType(bytes(type_frame.buffer))
285
286 # Deserialize the request data.
280 - decoder = add_request_decoder if (
281 - request_type
282 - == EngineCoreRequestType.ADD) else generic_decoder
287 + decoder = None
288 + if request_type == EngineCoreRequestType.ADD:
289 + decoder = add_request_decoder
290 + elif request_type == EngineCoreRequestType.ADD_LORA:
291 + decoder = add_lora_decoder
292 + else:
293 + decoder = generic_decoder
294 +
295 request = decoder.decode(data_frame.buffer)
296
297 # Push to input queue for core busy loop.
vllm/v1/engine/core_client.py
+16
@@ -12,6 +12,7 @@ import zmq.asyncio
12
13 from vllm.config import VllmConfig
14 from vllm.logger import init_logger
15 +from vllm.lora.request import LoRARequest
16 from vllm.utils import (get_open_zmq_ipc_path, kill_process_tree,
17 make_zmq_socket)
18 from vllm.v1.engine import (EngineCoreOutputs, EngineCoreRequest,
@@ -77,6 +78,9 @@ class EngineCoreClient(ABC):
78 def abort_requests(self, request_ids: List[str]) -> None:
79 raise NotImplementedError
80
81 + def add_lora(self, lora_request: LoRARequest) -> None:
82 + raise NotImplementedError
83 +
84 async def get_output_async(self) -> EngineCoreOutputs:
85 raise NotImplementedError
86
@@ -92,6 +96,9 @@ class EngineCoreClient(ABC):
96 async def abort_requests_async(self, request_ids: List[str]) -> None:
97 raise NotImplementedError
98
99 + async def add_lora_async(self, lora_request: LoRARequest) -> None:
100 + raise NotImplementedError
101 +
102
103 class InprocClient(EngineCoreClient):
104 """
@@ -125,6 +132,9 @@ class InprocClient(EngineCoreClient):
132 def reset_prefix_cache(self) -> None:
133 self.engine_core.reset_prefix_cache()
134
135 + def add_lora(self, lora_request: LoRARequest) -> None:
136 + self.engine_core.add_lora(lora_request)
137 +
138
139 class MPClient(EngineCoreClient):
140 """
@@ -242,6 +252,9 @@ class SyncMPClient(MPClient):
252 def reset_prefix_cache(self) -> None:
253 self._send_input(EngineCoreRequestType.RESET_PREFIX_CACHE, None)
254
255 + def add_lora(self, lora_request: LoRARequest) -> None:
256 + self._send_input(EngineCoreRequestType.ADD_LORA, lora_request)
257 +
258
259 class AsyncMPClient(MPClient):
260 """Asyncio-compatible client for multi-proc EngineCore."""
@@ -295,3 +308,6 @@ class AsyncMPClient(MPClient):
308
309 async def reset_prefix_cache_async(self) -> None:
310 await self._send_input(EngineCoreRequestType.RESET_PREFIX_CACHE, None)
311 +
312 + async def add_lora_async(self, lora_request: LoRARequest) -> None:
313 + await self._send_input(EngineCoreRequestType.ADD_LORA, lora_request)
vllm/v1/worker/gpu_worker.py
+4
@@ -15,6 +15,7 @@ from vllm.distributed import (ensure_model_parallel_initialized,
15 init_distributed_environment,
16 set_custom_all_reduce)
17 from vllm.logger import init_logger
18 +from vllm.lora.request import LoRARequest
19 from vllm.model_executor import set_random_seed
20 from vllm.platforms import current_platform
21 from vllm.utils import GiB_bytes
@@ -234,6 +235,9 @@ class Worker(WorkerBase):
235 else:
236 self.profiler.stop()
237
238 + def add_lora(self, lora_request: LoRARequest) -> bool:
239 + return self.model_runner.add_lora(lora_request)
240 +
241 def check_health(self) -> None:
242 # worker will always be healthy as long as it's running.
243 return
vllm/v1/worker/lora_model_runner_mixin.py
+5
@@ -127,3 +127,8 @@ class LoRAModelRunnerMixin:
127
128 # __exit__ code
129 self.lora_manager.remove_all_adapters()
130 +
131 + def add_lora(self, lora_request: LoRARequest) -> bool:
132 + if not self.lora_manager:
133 + raise RuntimeError("LoRA is not enabled.")
134 + return self.lora_manager.add_adapter(lora_request)
\ No newline at end of file