A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Core] Faster startup for LoRA enabled models (#4634)
Antoni Baum committed
May 8, 2024 at 10:33 UTC
ad932a221d2a4c1e6355021bb9e9c47f7a179e51
3 files changed
+47
-18
vllm/lora/models.py
+10
@@ -119,6 +119,16 @@ class LoRAModel:
119
self.rank = rank
120
self.loras: Dict[str, LoRALayerWeights] = loras
121
122
+ def clone(self, lora_model_id: int) -> "LoRAModel":
123
+ """Return a copy of the object with different ids.
124
+
125
+ Will share the underlying tensors."""
126
+ return self.__class__(
127
+ lora_model_id,
128
+ rank=self.rank,
129
+ loras=self.loras.copy(),
130
+ )
131
+
132
@property
133
def extra_vocab_size(self) -> int:
134
return max(lora.extra_vocab_size
vllm/lora/worker_manager.py
+22
-4
@@ -1,5 +1,6 @@
1
from abc import ABC, abstractmethod, abstractproperty
2
-from typing import Any, Dict, List, Set, Type
2
+from contextlib import contextmanager
3
+from typing import Any, Dict, List, Literal, Set, Type, Union
4
5
import torch
6
@@ -25,6 +26,17 @@ class AbstractWorkerLoRAManager(ABC):
26
self.device = device
27
self.lora_config = lora_config
28
29
+ # If False, do not cache. If None, cache is empty.
30
+ self._cached_dummy_lora: Union[None, Literal[False], LoRAModel] = False
31
+
32
+ @contextmanager
33
+ def dummy_lora_cache(self):
34
+ """Use this context manager to reuse the dummy lora model
35
+ to avoid creating it repeatedly."""
36
+ self._cached_dummy_lora = None
37
+ yield
38
+ self._cached_dummy_lora = False
39
+
40
@abstractproperty
41
def is_enabled(self) -> bool:
42
...
@@ -174,9 +186,15 @@ class WorkerLoRAManager(AbstractWorkerLoRAManager):
186
def add_dummy_lora(self, lora_request: LoRARequest, rank: int) -> bool:
187
if lora_request.lora_int_id in self.list_loras():
188
return False
177
- return self._lora_manager.add_lora(
178
- self._lora_manager.create_dummy_lora(lora_request.lora_int_id,
179
- rank, self.embedding_modules))
189
+ if isinstance(self._cached_dummy_lora, LoRAModel):
190
+ dummy_lora = self._cached_dummy_lora.clone(
191
+ lora_request.lora_int_id)
192
+ else:
193
+ dummy_lora = self._lora_manager.create_dummy_lora(
194
+ lora_request.lora_int_id, rank, self.embedding_modules)
195
+ if self._cached_dummy_lora is None:
196
+ self._cached_dummy_lora = dummy_lora
197
+ return self._lora_manager.add_lora(dummy_lora)
198
199
def add_lora(self, lora_request: LoRARequest) -> bool:
200
if lora_request.lora_int_id in self.list_loras():
vllm/worker/model_runner.py
+15
-14
@@ -835,20 +835,21 @@ class ModelRunner:
835
dummy_lora_requests = []
836
dummy_lora_requests_per_seq = []
837
if self.lora_config:
838
- for idx in range(self.lora_config.max_loras):
839
- lora_id = idx + 1
840
- dummy_lora_request = LoRARequest(
841
- lora_name=f"warmup_{lora_id}",
842
- lora_int_id=lora_id,
843
- lora_local_path="/not/a/real/path",
844
- )
845
- self.lora_manager.add_dummy_lora(dummy_lora_request,
846
- rank=LORA_WARMUP_RANK)
847
- dummy_lora_requests.append(dummy_lora_request)
848
- dummy_lora_requests_per_seq = [
849
- dummy_lora_requests[idx % len(dummy_lora_requests)]
850
- for idx in range(max_num_seqs)
851
- ]
838
+ with self.lora_manager.dummy_lora_cache():
839
+ for idx in range(self.lora_config.max_loras):
840
+ lora_id = idx + 1
841
+ dummy_lora_request = LoRARequest(
842
+ lora_name=f"warmup_{lora_id}",
843
+ lora_int_id=lora_id,
844
+ lora_local_path="/not/a/real/path",
845
+ )
846
+ self.lora_manager.add_dummy_lora(dummy_lora_request,
847
+ rank=LORA_WARMUP_RANK)
848
+ dummy_lora_requests.append(dummy_lora_request)
849
+ dummy_lora_requests_per_seq = [
850
+ dummy_lora_requests[idx % len(dummy_lora_requests)]
851
+ for idx in range(max_num_seqs)
852
+ ]
853
854
# Profile memory usage with max_num_sequences sequences and the total
855
# number of tokens equal to max_num_batched_tokens.