@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ad932a221d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Core] Faster startup for LoRA enabled models (#4634)

Antoni Baum committed May 8, 2024 at 10:33 UTC ad932a221d2a4c1e6355021bb9e9c47f7a179e51
3 files changed +47 -18
vllm/lora/models.py
+10
@@ -119,6 +119,16 @@ class LoRAModel:
119 self.rank = rank
120 self.loras: Dict[str, LoRALayerWeights] = loras
121
122 + def clone(self, lora_model_id: int) -> "LoRAModel":
123 + """Return a copy of the object with different ids.
124 +
125 + Will share the underlying tensors."""
126 + return self.__class__(
127 + lora_model_id,
128 + rank=self.rank,
129 + loras=self.loras.copy(),
130 + )
131 +
132 @property
133 def extra_vocab_size(self) -> int:
134 return max(lora.extra_vocab_size
vllm/lora/worker_manager.py
+22 -4
@@ -1,5 +1,6 @@
1 from abc import ABC, abstractmethod, abstractproperty
2 -from typing import Any, Dict, List, Set, Type
2 +from contextlib import contextmanager
3 +from typing import Any, Dict, List, Literal, Set, Type, Union
4
5 import torch
6
@@ -25,6 +26,17 @@ class AbstractWorkerLoRAManager(ABC):
26 self.device = device
27 self.lora_config = lora_config
28
29 + # If False, do not cache. If None, cache is empty.
30 + self._cached_dummy_lora: Union[None, Literal[False], LoRAModel] = False
31 +
32 + @contextmanager
33 + def dummy_lora_cache(self):
34 + """Use this context manager to reuse the dummy lora model
35 + to avoid creating it repeatedly."""
36 + self._cached_dummy_lora = None
37 + yield
38 + self._cached_dummy_lora = False
39 +
40 @abstractproperty
41 def is_enabled(self) -> bool:
42 ...
@@ -174,9 +186,15 @@ class WorkerLoRAManager(AbstractWorkerLoRAManager):
186 def add_dummy_lora(self, lora_request: LoRARequest, rank: int) -> bool:
187 if lora_request.lora_int_id in self.list_loras():
188 return False
177 - return self._lora_manager.add_lora(
178 - self._lora_manager.create_dummy_lora(lora_request.lora_int_id,
179 - rank, self.embedding_modules))
189 + if isinstance(self._cached_dummy_lora, LoRAModel):
190 + dummy_lora = self._cached_dummy_lora.clone(
191 + lora_request.lora_int_id)
192 + else:
193 + dummy_lora = self._lora_manager.create_dummy_lora(
194 + lora_request.lora_int_id, rank, self.embedding_modules)
195 + if self._cached_dummy_lora is None:
196 + self._cached_dummy_lora = dummy_lora
197 + return self._lora_manager.add_lora(dummy_lora)
198
199 def add_lora(self, lora_request: LoRARequest) -> bool:
200 if lora_request.lora_int_id in self.list_loras():
vllm/worker/model_runner.py
+15 -14
@@ -835,20 +835,21 @@ class ModelRunner:
835 dummy_lora_requests = []
836 dummy_lora_requests_per_seq = []
837 if self.lora_config:
838 - for idx in range(self.lora_config.max_loras):
839 - lora_id = idx + 1
840 - dummy_lora_request = LoRARequest(
841 - lora_name=f"warmup_{lora_id}",
842 - lora_int_id=lora_id,
843 - lora_local_path="/not/a/real/path",
844 - )
845 - self.lora_manager.add_dummy_lora(dummy_lora_request,
846 - rank=LORA_WARMUP_RANK)
847 - dummy_lora_requests.append(dummy_lora_request)
848 - dummy_lora_requests_per_seq = [
849 - dummy_lora_requests[idx % len(dummy_lora_requests)]
850 - for idx in range(max_num_seqs)
851 - ]
838 + with self.lora_manager.dummy_lora_cache():
839 + for idx in range(self.lora_config.max_loras):
840 + lora_id = idx + 1
841 + dummy_lora_request = LoRARequest(
842 + lora_name=f"warmup_{lora_id}",
843 + lora_int_id=lora_id,
844 + lora_local_path="/not/a/real/path",
845 + )
846 + self.lora_manager.add_dummy_lora(dummy_lora_request,
847 + rank=LORA_WARMUP_RANK)
848 + dummy_lora_requests.append(dummy_lora_request)
849 + dummy_lora_requests_per_seq = [
850 + dummy_lora_requests[idx % len(dummy_lora_requests)]
851 + for idx in range(max_num_seqs)
852 + ]
853
854 # Profile memory usage with max_num_sequences sequences and the total
855 # number of tokens equal to max_num_batched_tokens.