@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f5dda63eb5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[LoRA] Add support for pinning lora adapters in the LRU cache (#5603)

rohithkrn committed Jun 21, 2024 at 15:42 UTC f5dda63eb5fcb5624b93fa5f09da01d5372bbce4
13 files changed +171 -5
tests/lora/test_lora_manager.py
+64
@@ -209,6 +209,34 @@ def test_lora_lru_cache_model_manager(dist_init, dummy_model):
209 assert manager.activate_lora(3)
210 assert manager.lora_index_to_id[0] == 2
211 assert manager.lora_index_to_id[1] == 3
212 + assert manager.pin_lora(2)
213 + assert manager.lora_index_to_id[0] == 2
214 + assert manager.lora_index_to_id[1] == 3
215 + assert manager.activate_lora(1)
216 + assert manager.lora_index_to_id[0] == 2
217 + assert manager.lora_index_to_id[1] == 1
218 + assert manager.deactivate_lora(2)
219 + assert manager.lora_index_to_id[0] is None
220 + assert manager.lora_index_to_id[1] == 1
221 + assert manager.activate_lora(3)
222 + assert manager.lora_index_to_id[0] == 3
223 + assert manager.lora_index_to_id[1] == 1
224 + assert manager.pin_lora(3)
225 + assert manager.pin_lora(1)
226 + with pytest.raises(RuntimeError):
227 + assert manager.pin_lora(2)
228 + assert manager.lora_index_to_id[0] == 3
229 + assert manager.lora_index_to_id[1] == 1
230 + with pytest.raises(RuntimeError):
231 + assert manager.activate_lora(2)
232 +
233 + assert manager.deactivate_lora(3)
234 + assert manager.pin_lora(2)
235 + assert manager.lora_index_to_id[0] == 2
236 + assert manager.lora_index_to_id[1] == 1
237 + assert manager.remove_lora(3)
238 + with pytest.raises(ValueError):
239 + assert manager.pin_lora(3)
240
241
242 def test_lru_lora_model_manager(dist_init, dummy_model):
@@ -288,6 +316,42 @@ def test_lru_lora_model_manager(dist_init, dummy_model):
316 assert set(manager.list_loras()) == set()
317 assert all(x is None for x in manager.lora_index_to_id)
318
319 + # pinning
320 + assert manager.add_lora(model_lora3)
321 + assert manager.activate_lora(3)
322 + assert manager.add_lora(model_lora4)
323 + assert manager.activate_lora(4)
324 + assert set(manager.list_loras()) == {3, 4}
325 + with pytest.raises(ValueError):
326 + assert manager.pin_lora(1)
327 + assert manager.pin_lora(3)
328 + # Remove manually
329 + assert manager.remove_lora(3)
330 + assert not manager.remove_lora(3)
331 +
332 + assert set(manager.list_loras()) == {4}
333 + assert manager.lora_index_to_id[0] is None
334 + assert manager.lora_index_to_id[1] == 4
335 +
336 + assert manager.add_lora(model_lora1)
337 + assert manager.pin_lora(1)
338 + assert manager.add_lora(model_lora2)
339 + assert manager.activate_lora(2)
340 +
341 + assert set(manager.list_loras()) == {1, 2}
342 + assert manager.lora_index_to_id[0] == 1
343 + assert manager.lora_index_to_id[1] == 2
344 +
345 + assert manager.remove_oldest_lora()
346 + assert set(manager.list_loras()) == {1}
347 + assert manager.lora_index_to_id[0] == 1
348 + assert manager.lora_index_to_id[1] is None
349 +
350 + with pytest.raises(RuntimeError):
351 + assert manager.remove_oldest_lora()
352 +
353 + assert set(manager.list_loras()) == {1}
354 +
355
356 def test_lru_cache_worker_lora_manager(llama_2_7b_model_extra_embeddings,
357 sql_lora_files):
vllm/engine/llm_engine.py
+3
@@ -1009,6 +1009,9 @@ class LLMEngine:
1009 def list_loras(self) -> Set[int]:
1010 return self.model_executor.list_loras()
1011
1012 + def pin_lora(self, lora_id: int) -> bool:
1013 + return self.model_executor.pin_lora(lora_id)
1014 +
1015 def check_health(self) -> None:
1016 self.model_executor.check_health()
1017
vllm/executor/cpu_executor.py
+3
@@ -84,6 +84,9 @@ class CPUExecutor(ExecutorBase):
84 def remove_lora(self, lora_id: int) -> bool:
85 return self.driver_worker.remove_lora(lora_id)
86
87 + def pin_lora(self, lora_id: int) -> bool:
88 + return self.driver_worker.pin_lora(lora_id)
89 +
90 def list_loras(self) -> Set[int]:
91 return self.driver_worker.list_loras()
92
vllm/executor/distributed_gpu_executor.py
+7
@@ -100,6 +100,13 @@ class DistributedGPUExecutor(GPUExecutor):
100 lora_id=lora_id,
101 )
102
103 + def pin_lora(self, lora_id: int) -> bool:
104 + assert lora_id > 0, "lora_id must be greater than 0."
105 + return self._run_workers(
106 + "pin_lora",
107 + lora_id=lora_id,
108 + )
109 +
110 def list_loras(self) -> Set[int]:
111 return self._run_workers("list_loras")
112
vllm/executor/executor_base.py
+4
@@ -86,6 +86,10 @@ class ExecutorBase(ABC):
86 def remove_lora(self, lora_id: int) -> bool:
87 raise NotImplementedError
88
89 + @abstractmethod
90 + def pin_lora(self, lora_id: int) -> bool:
91 + raise NotImplementedError # type: ignore
92 +
93 @abstractmethod
94 def list_loras(self) -> Set[int]:
95 raise NotImplementedError
vllm/executor/gpu_executor.py
+4
@@ -99,6 +99,10 @@ class GPUExecutor(ExecutorBase):
99 assert lora_id > 0, "lora_id must be greater than 0."
100 return self.driver_worker.remove_lora(lora_id)
101
102 + def pin_lora(self, lora_id: int) -> bool:
103 + assert lora_id > 0, "lora_id must be greater than 0."
104 + return self.driver_worker.pin_lora(lora_id)
105 +
106 def list_loras(self) -> Set[int]:
107 return self.driver_worker.list_loras()
108
vllm/executor/neuron_executor.py
+3
@@ -65,6 +65,9 @@ class NeuronExecutor(ExecutorBase):
65 def remove_lora(self, lora_id: int) -> bool:
66 return self.driver_worker.remove_lora(lora_id)
67
68 + def pin_lora(self, lora_id: int) -> bool:
69 + return self.driver_worker.pin_lora(lora_id)
70 +
71 def list_loras(self) -> Set[int]:
72 return self.driver_worker.list_loras()
73
vllm/lora/models.py
+26
@@ -525,6 +525,12 @@ class LoRAModelManager:
525 self.long_lora_context.offsets_by_lora_id.pop(lora_id, None)
526 return bool(self._registered_loras.pop(lora_id, None))
527
528 + def pin_lora(self, lora_id: int) -> bool:
529 + """Pin a LoRAModel in the manager cache."""
530 + raise NotImplementedError(
531 + "Pinning is not supported in LoRAModelManager."
532 + "Use LRUCacheLoRAModelManager for pinning") # type: ignore
533 +
534 # TODO see if this can be vectorized
535 def _set_lora_mapping(self, mapping: LoRAMapping) -> None:
536 (base_indices, sampler_indices, sampler_indices_padded,
@@ -777,6 +783,26 @@ class LRUCacheLoRAModelManager(LoRAModelManager):
783 return True
784 return False
785
786 + def pin_lora(self, lora_id: int) -> bool:
787 + """Pin a LoRAModel in the manager cache."""
788 + self._pin_lora_in_cpu_cache(lora_id)
789 + self._pin_lora_in_gpu_cache(lora_id)
790 + return True
791 +
792 + def _pin_lora_in_cpu_cache(self, lora_id: int):
793 + try:
794 + self._registered_loras.pin(lora_id)
795 + except ValueError as err:
796 + raise ValueError("Pinning failed. "
797 + f"LoRA {lora_id} is not registered.") from err
798 +
799 + def _pin_lora_in_gpu_cache(self, lora_id: int):
800 + if lora_id not in self._active_loras:
801 + # move lora to gpu if not already active
802 + self.activate_lora(lora_id)
803 +
804 + self._active_loras.pin(lora_id)
805 +
806
807 def create_lora_manager(
808 model: nn.Module,
vllm/lora/worker_manager.py
+3
@@ -221,6 +221,9 @@ class WorkerLoRAManager(AbstractWorkerLoRAManager):
221 def remove_lora(self, lora_id: int) -> bool:
222 return self._lora_manager.remove_lora(lora_id)
223
224 + def pin_lora(self, lora_id: int) -> bool:
225 + return self._lora_manager.pin_lora(lora_id)
226 +
227 def remove_all_loras(self):
228 self._lora_manager.remove_all_loras()
229
vllm/utils.py
+38 -5
@@ -15,7 +15,7 @@ from collections import defaultdict
15 from functools import lru_cache, partial, wraps
16 from platform import uname
17 from typing import (Any, AsyncIterator, Awaitable, Callable, Dict, Generic,
18 - Hashable, List, Optional, OrderedDict, Tuple, TypeVar,
18 + Hashable, List, Optional, OrderedDict, Set, Tuple, TypeVar,
19 Union)
20
21 import numpy as np
@@ -44,6 +44,13 @@ K = TypeVar("K")
44 T = TypeVar("T")
45
46
47 +class _Sentinel:
48 + ...
49 +
50 +
51 +ALL_PINNED_SENTINEL = _Sentinel()
52 +
53 +
54 class Device(enum.Enum):
55 GPU = enum.auto()
56 CPU = enum.auto()
@@ -67,6 +74,7 @@ class LRUCache(Generic[T]):
74
75 def __init__(self, capacity: int):
76 self.cache: OrderedDict[Hashable, T] = OrderedDict()
77 + self.pinned_items: Set[Hashable] = set()
78 self.capacity = capacity
79
80 def __contains__(self, key: Hashable) -> bool:
@@ -102,14 +110,36 @@ class LRUCache(Generic[T]):
110 self.cache.move_to_end(key)
111 self._remove_old_if_needed()
112
113 + def pin(self, key: Hashable) -> None:
114 + """
115 + Pins a key in the cache preventing it from being
116 + evicted in the LRU order.
117 + """
118 + if key not in self.cache:
119 + raise ValueError(f"Cannot pin key: {key} not in cache.")
120 + self.pinned_items.add(key)
121 +
122 + def _unpin(self, key: Hashable) -> None:
123 + self.pinned_items.remove(key)
124 +
125 def _on_remove(self, key: Hashable, value: Optional[T]):
126 pass
127
108 - def remove_oldest(self):
128 + def remove_oldest(self, remove_pinned=False):
129 if not self.cache:
130 return
111 - key, value = self.cache.popitem(last=False)
112 - self._on_remove(key, value)
131 +
132 + if not remove_pinned:
133 + # pop the oldest item in the cache that is not pinned
134 + lru_key = next(
135 + (key for key in self.cache if key not in self.pinned_items),
136 + ALL_PINNED_SENTINEL)
137 + if lru_key is ALL_PINNED_SENTINEL:
138 + raise RuntimeError("All items are pinned, "
139 + "cannot remove oldest from the cache.")
140 + else:
141 + lru_key = next(iter(self.cache))
142 + self.pop(lru_key)
143
144 def _remove_old_if_needed(self) -> None:
145 while len(self.cache) > self.capacity:
@@ -120,13 +150,16 @@ class LRUCache(Generic[T]):
150 default_value: Optional[T] = None) -> Optional[T]:
151 run_on_remove = key in self.cache
152 value: Optional[T] = self.cache.pop(key, default_value)
153 + # remove from pinned items
154 + if key in self.pinned_items:
155 + self._unpin(key)
156 if run_on_remove:
157 self._on_remove(key, value)
158 return value
159
160 def clear(self):
161 while len(self.cache) > 0:
129 - self.remove_oldest()
162 + self.remove_oldest(remove_pinned=True)
163 self.cache.clear()
164
165
vllm/worker/model_runner.py
+5
@@ -878,6 +878,11 @@ class ModelRunner:
878 raise RuntimeError("LoRA is not enabled.")
879 return self.lora_manager.remove_lora(lora_id)
880
881 + def pin_lora(self, lora_id: int) -> bool:
882 + if not self.lora_manager:
883 + raise RuntimeError("LoRA is not enabled.")
884 + return self.lora_manager.pin_lora(lora_id)
885 +
886 def list_loras(self) -> Set[int]:
887 if not self.lora_manager:
888 raise RuntimeError("LoRA is not enabled.")
vllm/worker/worker.py
+3
@@ -333,6 +333,9 @@ class Worker(WorkerBase):
333 def remove_lora(self, lora_id: int) -> bool:
334 return self.model_runner.remove_lora(lora_id)
335
336 + def pin_lora(self, lora_id: int) -> bool:
337 + return self.model_runner.pin_lora(lora_id)
338 +
339 def list_loras(self) -> Set[int]:
340 return self.model_runner.list_loras()
341
vllm/worker/worker_base.py
+8
@@ -70,6 +70,10 @@ class WorkerBase(ABC):
70 def remove_lora(self, lora_id: int) -> bool:
71 raise NotImplementedError
72
73 + @abstractmethod
74 + def pin_lora(self, lora_id: int) -> bool:
75 + raise NotImplementedError
76 +
77 @abstractmethod
78 def list_loras(self) -> Set[int]:
79 raise NotImplementedError
@@ -86,6 +90,10 @@ class LoraNotSupportedWorkerBase(WorkerBase):
90 def remove_lora(self, lora_id: int) -> bool:
91 raise ValueError(f"{type(self)} does not support LoRA")
92
93 + def pin_lora(self, lora_id: int) -> bool:
94 + return ValueError(
95 + f"{type(self)} does not support LoRA") # type: ignore
96 +
97 def list_loras(self) -> Set[int]:
98 raise ValueError(f"{type(self)} does not support LoRA")
99