A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[LoRA] Add support for pinning lora adapters in the LRU cache (#5603)
rohithkrn committed
Jun 21, 2024 at 15:42 UTC
f5dda63eb5fcb5624b93fa5f09da01d5372bbce4
13 files changed
+171
-5
tests/lora/test_lora_manager.py
+64
@@ -209,6 +209,34 @@ def test_lora_lru_cache_model_manager(dist_init, dummy_model):
209
assert manager.activate_lora(3)
210
assert manager.lora_index_to_id[0] == 2
211
assert manager.lora_index_to_id[1] == 3
212
+ assert manager.pin_lora(2)
213
+ assert manager.lora_index_to_id[0] == 2
214
+ assert manager.lora_index_to_id[1] == 3
215
+ assert manager.activate_lora(1)
216
+ assert manager.lora_index_to_id[0] == 2
217
+ assert manager.lora_index_to_id[1] == 1
218
+ assert manager.deactivate_lora(2)
219
+ assert manager.lora_index_to_id[0] is None
220
+ assert manager.lora_index_to_id[1] == 1
221
+ assert manager.activate_lora(3)
222
+ assert manager.lora_index_to_id[0] == 3
223
+ assert manager.lora_index_to_id[1] == 1
224
+ assert manager.pin_lora(3)
225
+ assert manager.pin_lora(1)
226
+ with pytest.raises(RuntimeError):
227
+ assert manager.pin_lora(2)
228
+ assert manager.lora_index_to_id[0] == 3
229
+ assert manager.lora_index_to_id[1] == 1
230
+ with pytest.raises(RuntimeError):
231
+ assert manager.activate_lora(2)
232
+
233
+ assert manager.deactivate_lora(3)
234
+ assert manager.pin_lora(2)
235
+ assert manager.lora_index_to_id[0] == 2
236
+ assert manager.lora_index_to_id[1] == 1
237
+ assert manager.remove_lora(3)
238
+ with pytest.raises(ValueError):
239
+ assert manager.pin_lora(3)
240
241
242
def test_lru_lora_model_manager(dist_init, dummy_model):
@@ -288,6 +316,42 @@ def test_lru_lora_model_manager(dist_init, dummy_model):
316
assert set(manager.list_loras()) == set()
317
assert all(x is None for x in manager.lora_index_to_id)
318
319
+ # pinning
320
+ assert manager.add_lora(model_lora3)
321
+ assert manager.activate_lora(3)
322
+ assert manager.add_lora(model_lora4)
323
+ assert manager.activate_lora(4)
324
+ assert set(manager.list_loras()) == {3, 4}
325
+ with pytest.raises(ValueError):
326
+ assert manager.pin_lora(1)
327
+ assert manager.pin_lora(3)
328
+ # Remove manually
329
+ assert manager.remove_lora(3)
330
+ assert not manager.remove_lora(3)
331
+
332
+ assert set(manager.list_loras()) == {4}
333
+ assert manager.lora_index_to_id[0] is None
334
+ assert manager.lora_index_to_id[1] == 4
335
+
336
+ assert manager.add_lora(model_lora1)
337
+ assert manager.pin_lora(1)
338
+ assert manager.add_lora(model_lora2)
339
+ assert manager.activate_lora(2)
340
+
341
+ assert set(manager.list_loras()) == {1, 2}
342
+ assert manager.lora_index_to_id[0] == 1
343
+ assert manager.lora_index_to_id[1] == 2
344
+
345
+ assert manager.remove_oldest_lora()
346
+ assert set(manager.list_loras()) == {1}
347
+ assert manager.lora_index_to_id[0] == 1
348
+ assert manager.lora_index_to_id[1] is None
349
+
350
+ with pytest.raises(RuntimeError):
351
+ assert manager.remove_oldest_lora()
352
+
353
+ assert set(manager.list_loras()) == {1}
354
+
355
356
def test_lru_cache_worker_lora_manager(llama_2_7b_model_extra_embeddings,
357
sql_lora_files):
vllm/engine/llm_engine.py
+3
@@ -1009,6 +1009,9 @@ class LLMEngine:
1009
def list_loras(self) -> Set[int]:
1010
return self.model_executor.list_loras()
1011
1012
+ def pin_lora(self, lora_id: int) -> bool:
1013
+ return self.model_executor.pin_lora(lora_id)
1014
+
1015
def check_health(self) -> None:
1016
self.model_executor.check_health()
1017
vllm/executor/cpu_executor.py
+3
@@ -84,6 +84,9 @@ class CPUExecutor(ExecutorBase):
84
def remove_lora(self, lora_id: int) -> bool:
85
return self.driver_worker.remove_lora(lora_id)
86
87
+ def pin_lora(self, lora_id: int) -> bool:
88
+ return self.driver_worker.pin_lora(lora_id)
89
+
90
def list_loras(self) -> Set[int]:
91
return self.driver_worker.list_loras()
92
vllm/executor/distributed_gpu_executor.py
+7
@@ -100,6 +100,13 @@ class DistributedGPUExecutor(GPUExecutor):
100
lora_id=lora_id,
101
)
102
103
+ def pin_lora(self, lora_id: int) -> bool:
104
+ assert lora_id > 0, "lora_id must be greater than 0."
105
+ return self._run_workers(
106
+ "pin_lora",
107
+ lora_id=lora_id,
108
+ )
109
+
110
def list_loras(self) -> Set[int]:
111
return self._run_workers("list_loras")
112
vllm/executor/executor_base.py
+4
@@ -86,6 +86,10 @@ class ExecutorBase(ABC):
86
def remove_lora(self, lora_id: int) -> bool:
87
raise NotImplementedError
88
89
+ @abstractmethod
90
+ def pin_lora(self, lora_id: int) -> bool:
91
+ raise NotImplementedError # type: ignore
92
+
93
@abstractmethod
94
def list_loras(self) -> Set[int]:
95
raise NotImplementedError
vllm/executor/gpu_executor.py
+4
@@ -99,6 +99,10 @@ class GPUExecutor(ExecutorBase):
99
assert lora_id > 0, "lora_id must be greater than 0."
100
return self.driver_worker.remove_lora(lora_id)
101
102
+ def pin_lora(self, lora_id: int) -> bool:
103
+ assert lora_id > 0, "lora_id must be greater than 0."
104
+ return self.driver_worker.pin_lora(lora_id)
105
+
106
def list_loras(self) -> Set[int]:
107
return self.driver_worker.list_loras()
108
vllm/executor/neuron_executor.py
+3
@@ -65,6 +65,9 @@ class NeuronExecutor(ExecutorBase):
65
def remove_lora(self, lora_id: int) -> bool:
66
return self.driver_worker.remove_lora(lora_id)
67
68
+ def pin_lora(self, lora_id: int) -> bool:
69
+ return self.driver_worker.pin_lora(lora_id)
70
+
71
def list_loras(self) -> Set[int]:
72
return self.driver_worker.list_loras()
73
vllm/lora/models.py
+26
@@ -525,6 +525,12 @@ class LoRAModelManager:
525
self.long_lora_context.offsets_by_lora_id.pop(lora_id, None)
526
return bool(self._registered_loras.pop(lora_id, None))
527
528
+ def pin_lora(self, lora_id: int) -> bool:
529
+ """Pin a LoRAModel in the manager cache."""
530
+ raise NotImplementedError(
531
+ "Pinning is not supported in LoRAModelManager."
532
+ "Use LRUCacheLoRAModelManager for pinning") # type: ignore
533
+
534
# TODO see if this can be vectorized
535
def _set_lora_mapping(self, mapping: LoRAMapping) -> None:
536
(base_indices, sampler_indices, sampler_indices_padded,
@@ -777,6 +783,26 @@ class LRUCacheLoRAModelManager(LoRAModelManager):
783
return True
784
return False
785
786
+ def pin_lora(self, lora_id: int) -> bool:
787
+ """Pin a LoRAModel in the manager cache."""
788
+ self._pin_lora_in_cpu_cache(lora_id)
789
+ self._pin_lora_in_gpu_cache(lora_id)
790
+ return True
791
+
792
+ def _pin_lora_in_cpu_cache(self, lora_id: int):
793
+ try:
794
+ self._registered_loras.pin(lora_id)
795
+ except ValueError as err:
796
+ raise ValueError("Pinning failed. "
797
+ f"LoRA {lora_id} is not registered.") from err
798
+
799
+ def _pin_lora_in_gpu_cache(self, lora_id: int):
800
+ if lora_id not in self._active_loras:
801
+ # move lora to gpu if not already active
802
+ self.activate_lora(lora_id)
803
+
804
+ self._active_loras.pin(lora_id)
805
+
806
807
def create_lora_manager(
808
model: nn.Module,
vllm/lora/worker_manager.py
+3
@@ -221,6 +221,9 @@ class WorkerLoRAManager(AbstractWorkerLoRAManager):
221
def remove_lora(self, lora_id: int) -> bool:
222
return self._lora_manager.remove_lora(lora_id)
223
224
+ def pin_lora(self, lora_id: int) -> bool:
225
+ return self._lora_manager.pin_lora(lora_id)
226
+
227
def remove_all_loras(self):
228
self._lora_manager.remove_all_loras()
229
vllm/utils.py
+38
-5
@@ -15,7 +15,7 @@ from collections import defaultdict
15
from functools import lru_cache, partial, wraps
16
from platform import uname
17
from typing import (Any, AsyncIterator, Awaitable, Callable, Dict, Generic,
18
- Hashable, List, Optional, OrderedDict, Tuple, TypeVar,
18
+ Hashable, List, Optional, OrderedDict, Set, Tuple, TypeVar,
19
Union)
20
21
import numpy as np
@@ -44,6 +44,13 @@ K = TypeVar("K")
44
T = TypeVar("T")
45
46
47
+class _Sentinel:
48
+ ...
49
+
50
+
51
+ALL_PINNED_SENTINEL = _Sentinel()
52
+
53
+
54
class Device(enum.Enum):
55
GPU = enum.auto()
56
CPU = enum.auto()
@@ -67,6 +74,7 @@ class LRUCache(Generic[T]):
74
75
def __init__(self, capacity: int):
76
self.cache: OrderedDict[Hashable, T] = OrderedDict()
77
+ self.pinned_items: Set[Hashable] = set()
78
self.capacity = capacity
79
80
def __contains__(self, key: Hashable) -> bool:
@@ -102,14 +110,36 @@ class LRUCache(Generic[T]):
110
self.cache.move_to_end(key)
111
self._remove_old_if_needed()
112
113
+ def pin(self, key: Hashable) -> None:
114
+ """
115
+ Pins a key in the cache preventing it from being
116
+ evicted in the LRU order.
117
+ """
118
+ if key not in self.cache:
119
+ raise ValueError(f"Cannot pin key: {key} not in cache.")
120
+ self.pinned_items.add(key)
121
+
122
+ def _unpin(self, key: Hashable) -> None:
123
+ self.pinned_items.remove(key)
124
+
125
def _on_remove(self, key: Hashable, value: Optional[T]):
126
pass
127
108
- def remove_oldest(self):
128
+ def remove_oldest(self, remove_pinned=False):
129
if not self.cache:
130
return
111
- key, value = self.cache.popitem(last=False)
112
- self._on_remove(key, value)
131
+
132
+ if not remove_pinned:
133
+ # pop the oldest item in the cache that is not pinned
134
+ lru_key = next(
135
+ (key for key in self.cache if key not in self.pinned_items),
136
+ ALL_PINNED_SENTINEL)
137
+ if lru_key is ALL_PINNED_SENTINEL:
138
+ raise RuntimeError("All items are pinned, "
139
+ "cannot remove oldest from the cache.")
140
+ else:
141
+ lru_key = next(iter(self.cache))
142
+ self.pop(lru_key)
143
144
def _remove_old_if_needed(self) -> None:
145
while len(self.cache) > self.capacity:
@@ -120,13 +150,16 @@ class LRUCache(Generic[T]):
150
default_value: Optional[T] = None) -> Optional[T]:
151
run_on_remove = key in self.cache
152
value: Optional[T] = self.cache.pop(key, default_value)
153
+ # remove from pinned items
154
+ if key in self.pinned_items:
155
+ self._unpin(key)
156
if run_on_remove:
157
self._on_remove(key, value)
158
return value
159
160
def clear(self):
161
while len(self.cache) > 0:
129
- self.remove_oldest()
162
+ self.remove_oldest(remove_pinned=True)
163
self.cache.clear()
164
165
vllm/worker/model_runner.py
+5
@@ -878,6 +878,11 @@ class ModelRunner:
878
raise RuntimeError("LoRA is not enabled.")
879
return self.lora_manager.remove_lora(lora_id)
880
881
+ def pin_lora(self, lora_id: int) -> bool:
882
+ if not self.lora_manager:
883
+ raise RuntimeError("LoRA is not enabled.")
884
+ return self.lora_manager.pin_lora(lora_id)
885
+
886
def list_loras(self) -> Set[int]:
887
if not self.lora_manager:
888
raise RuntimeError("LoRA is not enabled.")
vllm/worker/worker.py
+3
@@ -333,6 +333,9 @@ class Worker(WorkerBase):
333
def remove_lora(self, lora_id: int) -> bool:
334
return self.model_runner.remove_lora(lora_id)
335
336
+ def pin_lora(self, lora_id: int) -> bool:
337
+ return self.model_runner.pin_lora(lora_id)
338
+
339
def list_loras(self) -> Set[int]:
340
return self.model_runner.list_loras()
341
vllm/worker/worker_base.py
+8
@@ -70,6 +70,10 @@ class WorkerBase(ABC):
70
def remove_lora(self, lora_id: int) -> bool:
71
raise NotImplementedError
72
73
+ @abstractmethod
74
+ def pin_lora(self, lora_id: int) -> bool:
75
+ raise NotImplementedError
76
+
77
@abstractmethod
78
def list_loras(self) -> Set[int]:
79
raise NotImplementedError
@@ -86,6 +90,10 @@ class LoraNotSupportedWorkerBase(WorkerBase):
90
def remove_lora(self, lora_id: int) -> bool:
91
raise ValueError(f"{type(self)} does not support LoRA")
92
93
+ def pin_lora(self, lora_id: int) -> bool:
94
+ return ValueError(
95
+ f"{type(self)} does not support LoRA") # type: ignore
96
+
97
def list_loras(self) -> Set[int]:
98
raise ValueError(f"{type(self)} does not support LoRA")
99