A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Misc] Clean up and consolidate LRUCache (#11339)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Dec 20, 2024 at 00:59 UTC
cdf22afddad7b29e8d584b77863a563a91ac09fb
5 files changed
+34
-67
vllm/adapter_commons/models.py
+4
-5
@@ -1,5 +1,5 @@
1
from abc import ABC, abstractmethod
2
-from typing import Any, Callable, Dict, Hashable, Optional, TypeVar
2
+from typing import Any, Callable, Dict, Optional, TypeVar
3
4
from torch import nn
5
@@ -24,14 +24,13 @@ class AdapterModel(ABC):
24
T = TypeVar('T')
25
26
27
-class AdapterLRUCache(LRUCache[T]):
27
+class AdapterLRUCache(LRUCache[int, T]):
28
29
- def __init__(self, capacity: int, deactivate_fn: Callable[[Hashable],
30
- None]):
29
+ def __init__(self, capacity: int, deactivate_fn: Callable[[int], object]):
30
super().__init__(capacity)
31
self.deactivate_fn = deactivate_fn
32
34
- def _on_remove(self, key: Hashable, value: Optional[T]):
33
+ def _on_remove(self, key: int, value: Optional[T]):
34
logger.debug("Removing adapter int id: %d", key)
35
self.deactivate_fn(key)
36
return super()._on_remove(key, value)
vllm/transformers_utils/tokenizer_group/tokenizer_group.py
+1
-1
@@ -22,7 +22,7 @@ class TokenizerGroup(BaseTokenizerGroup):
22
self.max_input_length = max_input_length
23
self.tokenizer = get_tokenizer(self.tokenizer_id, **tokenizer_config)
24
max_loras = tokenizer_config.get("max_loras", 0)
25
- self.lora_tokenizers = LRUCache[AnyTokenizer](
25
+ self.lora_tokenizers = LRUCache[int, AnyTokenizer](
26
capacity=max(max_loras, max_num_seqs) if enable_lora else 0)
27
28
@classmethod
vllm/utils.py
+26
-33
@@ -21,14 +21,13 @@ import uuid
21
import warnings
22
import weakref
23
from asyncio import FIRST_COMPLETED, AbstractEventLoop, Task
24
-from collections import UserDict, defaultdict
24
+from collections import OrderedDict, UserDict, defaultdict
25
from collections.abc import Iterable, Mapping
26
from dataclasses import dataclass, field
27
from functools import lru_cache, partial, wraps
28
from typing import (TYPE_CHECKING, Any, AsyncGenerator, Awaitable, Callable,
29
Dict, Generator, Generic, Hashable, List, Literal,
30
- Optional, OrderedDict, Set, Tuple, Type, TypeVar, Union,
31
- overload)
30
+ Optional, Tuple, Type, TypeVar, Union, overload)
31
from uuid import uuid4
32
33
import numpy as np
@@ -154,10 +153,12 @@ TORCH_DTYPE_TO_NUMPY_DTYPE = {
153
}
154
155
P = ParamSpec('P')
157
-K = TypeVar("K")
156
T = TypeVar("T")
157
U = TypeVar("U")
158
159
+_K = TypeVar("_K", bound=Hashable)
160
+_V = TypeVar("_V")
161
+
162
163
class _Sentinel:
164
...
@@ -190,50 +191,48 @@ class Counter:
191
self.counter = 0
192
193
193
-class LRUCache(Generic[T]):
194
+class LRUCache(Generic[_K, _V]):
195
195
- def __init__(self, capacity: int):
196
- self.cache: OrderedDict[Hashable, T] = OrderedDict()
197
- self.pinned_items: Set[Hashable] = set()
196
+ def __init__(self, capacity: int) -> None:
197
+ self.cache = OrderedDict[_K, _V]()
198
+ self.pinned_items = set[_K]()
199
self.capacity = capacity
200
200
- def __contains__(self, key: Hashable) -> bool:
201
+ def __contains__(self, key: _K) -> bool:
202
return key in self.cache
203
204
def __len__(self) -> int:
205
return len(self.cache)
206
206
- def __getitem__(self, key: Hashable) -> T:
207
+ def __getitem__(self, key: _K) -> _V:
208
value = self.cache[key] # Raise KeyError if not exists
209
self.cache.move_to_end(key)
210
return value
211
211
- def __setitem__(self, key: Hashable, value: T) -> None:
212
+ def __setitem__(self, key: _K, value: _V) -> None:
213
self.put(key, value)
214
214
- def __delitem__(self, key: Hashable) -> None:
215
+ def __delitem__(self, key: _K) -> None:
216
self.pop(key)
217
217
- def touch(self, key: Hashable) -> None:
218
+ def touch(self, key: _K) -> None:
219
self.cache.move_to_end(key)
220
220
- def get(self,
221
- key: Hashable,
222
- default_value: Optional[T] = None) -> Optional[T]:
223
- value: Optional[T]
221
+ def get(self, key: _K, default: Optional[_V] = None) -> Optional[_V]:
222
+ value: Optional[_V]
223
if key in self.cache:
224
value = self.cache[key]
225
self.cache.move_to_end(key)
226
else:
228
- value = default_value
227
+ value = default
228
return value
229
231
- def put(self, key: Hashable, value: T) -> None:
230
+ def put(self, key: _K, value: _V) -> None:
231
self.cache[key] = value
232
self.cache.move_to_end(key)
233
self._remove_old_if_needed()
234
236
- def pin(self, key: Hashable) -> None:
235
+ def pin(self, key: _K) -> None:
236
"""
237
Pins a key in the cache preventing it from being
238
evicted in the LRU order.
@@ -242,13 +241,13 @@ class LRUCache(Generic[T]):
241
raise ValueError(f"Cannot pin key: {key} not in cache.")
242
self.pinned_items.add(key)
243
245
- def _unpin(self, key: Hashable) -> None:
244
+ def _unpin(self, key: _K) -> None:
245
self.pinned_items.remove(key)
246
248
- def _on_remove(self, key: Hashable, value: Optional[T]):
247
+ def _on_remove(self, key: _K, value: Optional[_V]) -> None:
248
pass
249
251
- def remove_oldest(self, remove_pinned=False):
250
+ def remove_oldest(self, *, remove_pinned: bool = False) -> None:
251
if not self.cache:
252
return
253
@@ -262,17 +261,15 @@ class LRUCache(Generic[T]):
261
"cannot remove oldest from the cache.")
262
else:
263
lru_key = next(iter(self.cache))
265
- self.pop(lru_key)
264
+ self.pop(lru_key) # type: ignore
265
266
def _remove_old_if_needed(self) -> None:
267
while len(self.cache) > self.capacity:
268
self.remove_oldest()
269
271
- def pop(self,
272
- key: Hashable,
273
- default_value: Optional[T] = None) -> Optional[T]:
270
+ def pop(self, key: _K, default: Optional[_V] = None) -> Optional[_V]:
271
run_on_remove = key in self.cache
275
- value: Optional[T] = self.cache.pop(key, default_value)
272
+ value = self.cache.pop(key, default)
273
# remove from pinned items
274
if key in self.pinned_items:
275
self._unpin(key)
@@ -280,7 +277,7 @@ class LRUCache(Generic[T]):
277
self._on_remove(key, value)
278
return value
279
283
- def clear(self):
280
+ def clear(self) -> None:
281
while len(self.cache) > 0:
282
self.remove_oldest(remove_pinned=True)
283
self.cache.clear()
@@ -843,10 +840,6 @@ def flatten_2d_lists(lists: List[List[T]]) -> List[T]:
840
return [item for sublist in lists for item in sublist]
841
842
846
-_K = TypeVar("_K", bound=Hashable)
847
-_V = TypeVar("_V")
848
-
849
-
843
def full_groupby(values: Iterable[_V], *, key: Callable[[_V], _K]):
844
"""
845
Unlike :class:`itertools.groupby`, groups are not broken by
vllm/v1/engine/mm_input_mapper.py
+3
-3
@@ -8,7 +8,7 @@ from vllm.inputs import PromptType
8
from vllm.logger import init_logger
9
from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalDataDict,
10
MultiModalKwargs, MultiModalRegistry)
11
-from vllm.v1.utils import LRUDictCache
11
+from vllm.utils import LRUCache
12
13
logger = init_logger(__name__)
14
@@ -44,7 +44,7 @@ class MMInputMapperClient:
44
45
# Init cache
46
self.use_cache = not model_config.disable_mm_preprocessor_cache
47
- self.mm_cache = LRUDictCache[str, MultiModalKwargs](MM_CACHE_SIZE)
47
+ self.mm_cache = LRUCache[str, MultiModalKwargs](MM_CACHE_SIZE)
48
49
# DEBUG: Set to None to disable
50
self.mm_debug_cache_hit_ratio_steps = None
@@ -120,7 +120,7 @@ class MMInputMapperServer:
120
121
def __init__(self, model_config):
122
self.use_cache = not model_config.disable_mm_preprocessor_cache
123
- self.mm_cache = LRUDictCache[str, MultiModalKwargs](MM_CACHE_SIZE)
123
+ self.mm_cache = LRUCache[str, MultiModalKwargs](MM_CACHE_SIZE)
124
125
def process_inputs(
126
self,
vllm/v1/utils.py
-25
@@ -1,4 +1,3 @@
1
-from collections import OrderedDict
1
from collections.abc import Sequence
2
from contextlib import contextmanager
3
from typing import (Any, Generic, Iterator, List, Optional, TypeVar, Union,
@@ -102,27 +101,3 @@ def make_zmq_socket(
101
102
finally:
103
ctx.destroy(linger=0)
105
-
106
-
107
-K = TypeVar('K')
108
-V = TypeVar('V')
109
-
110
-
111
-class LRUDictCache(Generic[K, V]):
112
-
113
- def __init__(self, size: int):
114
- self.cache: OrderedDict[K, V] = OrderedDict()
115
- self.size = size
116
-
117
- def get(self, key: K, default=None) -> V:
118
- if key not in self.cache:
119
- return default
120
-
121
- self.cache.move_to_end(key)
122
- return self.cache[key]
123
-
124
- def put(self, key: K, value: V):
125
- self.cache[key] = value
126
- self.cache.move_to_end(key)
127
- if len(self.cache) > self.size:
128
- self.cache.popitem(last=False)