@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cdf22afdda
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Misc] Clean up and consolidate LRUCache (#11339)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Dec 20, 2024 at 00:59 UTC cdf22afddad7b29e8d584b77863a563a91ac09fb
5 files changed +34 -67
vllm/adapter_commons/models.py
+4 -5
@@ -1,5 +1,5 @@
1 from abc import ABC, abstractmethod
2 -from typing import Any, Callable, Dict, Hashable, Optional, TypeVar
2 +from typing import Any, Callable, Dict, Optional, TypeVar
3
4 from torch import nn
5
@@ -24,14 +24,13 @@ class AdapterModel(ABC):
24 T = TypeVar('T')
25
26
27 -class AdapterLRUCache(LRUCache[T]):
27 +class AdapterLRUCache(LRUCache[int, T]):
28
29 - def __init__(self, capacity: int, deactivate_fn: Callable[[Hashable],
30 - None]):
29 + def __init__(self, capacity: int, deactivate_fn: Callable[[int], object]):
30 super().__init__(capacity)
31 self.deactivate_fn = deactivate_fn
32
34 - def _on_remove(self, key: Hashable, value: Optional[T]):
33 + def _on_remove(self, key: int, value: Optional[T]):
34 logger.debug("Removing adapter int id: %d", key)
35 self.deactivate_fn(key)
36 return super()._on_remove(key, value)
vllm/transformers_utils/tokenizer_group/tokenizer_group.py
+1 -1
@@ -22,7 +22,7 @@ class TokenizerGroup(BaseTokenizerGroup):
22 self.max_input_length = max_input_length
23 self.tokenizer = get_tokenizer(self.tokenizer_id, **tokenizer_config)
24 max_loras = tokenizer_config.get("max_loras", 0)
25 - self.lora_tokenizers = LRUCache[AnyTokenizer](
25 + self.lora_tokenizers = LRUCache[int, AnyTokenizer](
26 capacity=max(max_loras, max_num_seqs) if enable_lora else 0)
27
28 @classmethod
vllm/utils.py
+26 -33
@@ -21,14 +21,13 @@ import uuid
21 import warnings
22 import weakref
23 from asyncio import FIRST_COMPLETED, AbstractEventLoop, Task
24 -from collections import UserDict, defaultdict
24 +from collections import OrderedDict, UserDict, defaultdict
25 from collections.abc import Iterable, Mapping
26 from dataclasses import dataclass, field
27 from functools import lru_cache, partial, wraps
28 from typing import (TYPE_CHECKING, Any, AsyncGenerator, Awaitable, Callable,
29 Dict, Generator, Generic, Hashable, List, Literal,
30 - Optional, OrderedDict, Set, Tuple, Type, TypeVar, Union,
31 - overload)
30 + Optional, Tuple, Type, TypeVar, Union, overload)
31 from uuid import uuid4
32
33 import numpy as np
@@ -154,10 +153,12 @@ TORCH_DTYPE_TO_NUMPY_DTYPE = {
153 }
154
155 P = ParamSpec('P')
157 -K = TypeVar("K")
156 T = TypeVar("T")
157 U = TypeVar("U")
158
159 +_K = TypeVar("_K", bound=Hashable)
160 +_V = TypeVar("_V")
161 +
162
163 class _Sentinel:
164 ...
@@ -190,50 +191,48 @@ class Counter:
191 self.counter = 0
192
193
193 -class LRUCache(Generic[T]):
194 +class LRUCache(Generic[_K, _V]):
195
195 - def __init__(self, capacity: int):
196 - self.cache: OrderedDict[Hashable, T] = OrderedDict()
197 - self.pinned_items: Set[Hashable] = set()
196 + def __init__(self, capacity: int) -> None:
197 + self.cache = OrderedDict[_K, _V]()
198 + self.pinned_items = set[_K]()
199 self.capacity = capacity
200
200 - def __contains__(self, key: Hashable) -> bool:
201 + def __contains__(self, key: _K) -> bool:
202 return key in self.cache
203
204 def __len__(self) -> int:
205 return len(self.cache)
206
206 - def __getitem__(self, key: Hashable) -> T:
207 + def __getitem__(self, key: _K) -> _V:
208 value = self.cache[key] # Raise KeyError if not exists
209 self.cache.move_to_end(key)
210 return value
211
211 - def __setitem__(self, key: Hashable, value: T) -> None:
212 + def __setitem__(self, key: _K, value: _V) -> None:
213 self.put(key, value)
214
214 - def __delitem__(self, key: Hashable) -> None:
215 + def __delitem__(self, key: _K) -> None:
216 self.pop(key)
217
217 - def touch(self, key: Hashable) -> None:
218 + def touch(self, key: _K) -> None:
219 self.cache.move_to_end(key)
220
220 - def get(self,
221 - key: Hashable,
222 - default_value: Optional[T] = None) -> Optional[T]:
223 - value: Optional[T]
221 + def get(self, key: _K, default: Optional[_V] = None) -> Optional[_V]:
222 + value: Optional[_V]
223 if key in self.cache:
224 value = self.cache[key]
225 self.cache.move_to_end(key)
226 else:
228 - value = default_value
227 + value = default
228 return value
229
231 - def put(self, key: Hashable, value: T) -> None:
230 + def put(self, key: _K, value: _V) -> None:
231 self.cache[key] = value
232 self.cache.move_to_end(key)
233 self._remove_old_if_needed()
234
236 - def pin(self, key: Hashable) -> None:
235 + def pin(self, key: _K) -> None:
236 """
237 Pins a key in the cache preventing it from being
238 evicted in the LRU order.
@@ -242,13 +241,13 @@ class LRUCache(Generic[T]):
241 raise ValueError(f"Cannot pin key: {key} not in cache.")
242 self.pinned_items.add(key)
243
245 - def _unpin(self, key: Hashable) -> None:
244 + def _unpin(self, key: _K) -> None:
245 self.pinned_items.remove(key)
246
248 - def _on_remove(self, key: Hashable, value: Optional[T]):
247 + def _on_remove(self, key: _K, value: Optional[_V]) -> None:
248 pass
249
251 - def remove_oldest(self, remove_pinned=False):
250 + def remove_oldest(self, *, remove_pinned: bool = False) -> None:
251 if not self.cache:
252 return
253
@@ -262,17 +261,15 @@ class LRUCache(Generic[T]):
261 "cannot remove oldest from the cache.")
262 else:
263 lru_key = next(iter(self.cache))
265 - self.pop(lru_key)
264 + self.pop(lru_key) # type: ignore
265
266 def _remove_old_if_needed(self) -> None:
267 while len(self.cache) > self.capacity:
268 self.remove_oldest()
269
271 - def pop(self,
272 - key: Hashable,
273 - default_value: Optional[T] = None) -> Optional[T]:
270 + def pop(self, key: _K, default: Optional[_V] = None) -> Optional[_V]:
271 run_on_remove = key in self.cache
275 - value: Optional[T] = self.cache.pop(key, default_value)
272 + value = self.cache.pop(key, default)
273 # remove from pinned items
274 if key in self.pinned_items:
275 self._unpin(key)
@@ -280,7 +277,7 @@ class LRUCache(Generic[T]):
277 self._on_remove(key, value)
278 return value
279
283 - def clear(self):
280 + def clear(self) -> None:
281 while len(self.cache) > 0:
282 self.remove_oldest(remove_pinned=True)
283 self.cache.clear()
@@ -843,10 +840,6 @@ def flatten_2d_lists(lists: List[List[T]]) -> List[T]:
840 return [item for sublist in lists for item in sublist]
841
842
846 -_K = TypeVar("_K", bound=Hashable)
847 -_V = TypeVar("_V")
848 -
849 -
843 def full_groupby(values: Iterable[_V], *, key: Callable[[_V], _K]):
844 """
845 Unlike :class:`itertools.groupby`, groups are not broken by
vllm/v1/engine/mm_input_mapper.py
+3 -3
@@ -8,7 +8,7 @@ from vllm.inputs import PromptType
8 from vllm.logger import init_logger
9 from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalDataDict,
10 MultiModalKwargs, MultiModalRegistry)
11 -from vllm.v1.utils import LRUDictCache
11 +from vllm.utils import LRUCache
12
13 logger = init_logger(__name__)
14
@@ -44,7 +44,7 @@ class MMInputMapperClient:
44
45 # Init cache
46 self.use_cache = not model_config.disable_mm_preprocessor_cache
47 - self.mm_cache = LRUDictCache[str, MultiModalKwargs](MM_CACHE_SIZE)
47 + self.mm_cache = LRUCache[str, MultiModalKwargs](MM_CACHE_SIZE)
48
49 # DEBUG: Set to None to disable
50 self.mm_debug_cache_hit_ratio_steps = None
@@ -120,7 +120,7 @@ class MMInputMapperServer:
120
121 def __init__(self, model_config):
122 self.use_cache = not model_config.disable_mm_preprocessor_cache
123 - self.mm_cache = LRUDictCache[str, MultiModalKwargs](MM_CACHE_SIZE)
123 + self.mm_cache = LRUCache[str, MultiModalKwargs](MM_CACHE_SIZE)
124
125 def process_inputs(
126 self,
vllm/v1/utils.py
-25
@@ -1,4 +1,3 @@
1 -from collections import OrderedDict
1 from collections.abc import Sequence
2 from contextlib import contextmanager
3 from typing import (Any, Generic, Iterator, List, Optional, TypeVar, Union,
@@ -102,27 +101,3 @@ def make_zmq_socket(
101
102 finally:
103 ctx.destroy(linger=0)
105 -
106 -
107 -K = TypeVar('K')
108 -V = TypeVar('V')
109 -
110 -
111 -class LRUDictCache(Generic[K, V]):
112 -
113 - def __init__(self, size: int):
114 - self.cache: OrderedDict[K, V] = OrderedDict()
115 - self.size = size
116 -
117 - def get(self, key: K, default=None) -> V:
118 - if key not in self.cache:
119 - return default
120 -
121 - self.cache.move_to_end(key)
122 - return self.cache[key]
123 -
124 - def put(self, key: K, value: V):
125 - self.cache[key] = value
126 - self.cache.move_to_end(key)
127 - if len(self.cache) > self.size:
128 - self.cache.popitem(last=False)