A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
[Core] Add generic typing to `LRUCache` (#3511)
Nick Hill committed
Mar 20, 2024 at 00:36 UTC
4ad521d8b51145a55c1be6b8e451f76423cc2d87
4 files changed
+29
-22
vllm/lora/models.py
+3
-3
@@ -4,7 +4,7 @@ import logging
4
import math
5
import os
6
import re
7
-from typing import (Any, Callable, Dict, Hashable, List, Optional, Tuple, Type)
7
+from typing import (Callable, Dict, Hashable, List, Optional, Tuple, Type)
8
9
import safetensors.torch
10
import torch
@@ -535,14 +535,14 @@ class LoRAModelManager:
535
replacement_loras)
536
537
538
-class LoRALRUCache(LRUCache):
538
+class LoRALRUCache(LRUCache[LoRAModel]):
539
540
def __init__(self, capacity: int, deactivate_lora_fn: Callable[[Hashable],
541
None]):
542
super().__init__(capacity)
543
self.deactivate_lora_fn = deactivate_lora_fn
544
545
- def _on_remove(self, key: Hashable, value: Any):
545
+ def _on_remove(self, key: Hashable, value: LoRAModel):
546
logger.debug(f"Removing LoRA. int id: {key}")
547
self.deactivate_lora_fn(key)
548
return super()._on_remove(key, value)
vllm/transformers_utils/tokenizer_group/base_tokenizer_group.py
+13
-6
@@ -22,27 +22,34 @@ class BaseTokenizerGroup(ABC):
22
pass
23
24
@abstractmethod
25
- def encode(self, prompt: str, request_id: Optional[str],
26
- lora_request: Optional[LoRARequest]) -> List[int]:
25
+ def encode(self,
26
+ prompt: str,
27
+ request_id: Optional[str] = None,
28
+ lora_request: Optional[LoRARequest] = None) -> List[int]:
29
"""Encode a prompt using the tokenizer group."""
30
pass
31
32
@abstractmethod
31
- async def encode_async(self, prompt: str, request_id: Optional[str],
32
- lora_request: Optional[LoRARequest]) -> List[int]:
33
+ async def encode_async(
34
+ self,
35
+ prompt: str,
36
+ request_id: Optional[str] = None,
37
+ lora_request: Optional[LoRARequest] = None) -> List[int]:
38
"""Encode a prompt using the tokenizer group."""
39
pass
40
41
@abstractmethod
42
def get_lora_tokenizer(
43
self,
39
- lora_request: Optional[LoRARequest]) -> "PreTrainedTokenizer":
44
+ lora_request: Optional[LoRARequest] = None
45
+ ) -> "PreTrainedTokenizer":
46
"""Get a tokenizer for a LoRA request."""
47
pass
48
49
@abstractmethod
50
async def get_lora_tokenizer_async(
51
self,
46
- lora_request: Optional[LoRARequest]) -> "PreTrainedTokenizer":
52
+ lora_request: Optional[LoRARequest] = None
53
+ ) -> "PreTrainedTokenizer":
54
"""Get a tokenizer for a LoRA request."""
55
pass
vllm/transformers_utils/tokenizer_group/tokenizer_group.py
+2
-4
@@ -21,10 +21,8 @@ class TokenizerGroup(BaseTokenizerGroup):
21
self.enable_lora = enable_lora
22
self.max_input_length = max_input_length
23
self.tokenizer = get_tokenizer(self.tokenizer_id, **tokenizer_config)
24
- if enable_lora:
25
- self.lora_tokenizers = LRUCache(capacity=max_num_seqs)
26
- else:
27
- self.lora_tokenizers = None
24
+ self.lora_tokenizers = LRUCache[PreTrainedTokenizer](
25
+ capacity=max_num_seqs) if enable_lora else None
26
27
def ping(self) -> bool:
28
"""Check if the tokenizer group is alive."""
vllm/utils.py
+11
-9
@@ -5,7 +5,7 @@ import subprocess
5
import uuid
6
import gc
7
from platform import uname
8
-from typing import List, Tuple, Union
8
+from typing import List, Tuple, Union, Generic
9
from packaging.version import parse, Version
10
11
import psutil
@@ -53,10 +53,10 @@ class Counter:
53
self.counter = 0
54
55
56
-class LRUCache:
56
+class LRUCache(Generic[T]):
57
58
def __init__(self, capacity: int):
59
- self.cache = OrderedDict()
59
+ self.cache = OrderedDict[Hashable, T]()
60
self.capacity = capacity
61
62
def __contains__(self, key: Hashable) -> bool:
@@ -65,10 +65,10 @@ class LRUCache:
65
def __len__(self) -> int:
66
return len(self.cache)
67
68
- def __getitem__(self, key: Hashable) -> Any:
68
+ def __getitem__(self, key: Hashable) -> T:
69
return self.get(key)
70
71
- def __setitem__(self, key: Hashable, value: Any) -> None:
71
+ def __setitem__(self, key: Hashable, value: T) -> None:
72
self.put(key, value)
73
74
def __delitem__(self, key: Hashable) -> None:
@@ -77,7 +77,9 @@ class LRUCache:
77
def touch(self, key: Hashable) -> None:
78
self.cache.move_to_end(key)
79
80
- def get(self, key: Hashable, default_value: Optional[Any] = None) -> int:
80
+ def get(self,
81
+ key: Hashable,
82
+ default_value: Optional[T] = None) -> Optional[T]:
83
if key in self.cache:
84
value = self.cache[key]
85
self.cache.move_to_end(key)
@@ -85,12 +87,12 @@ class LRUCache:
87
value = default_value
88
return value
89
88
- def put(self, key: Hashable, value: Any) -> None:
90
+ def put(self, key: Hashable, value: T) -> None:
91
self.cache[key] = value
92
self.cache.move_to_end(key)
93
self._remove_old_if_needed()
94
93
- def _on_remove(self, key: Hashable, value: Any):
95
+ def _on_remove(self, key: Hashable, value: T):
96
pass
97
98
def remove_oldest(self):
@@ -103,7 +105,7 @@ class LRUCache:
105
while len(self.cache) > self.capacity:
106
self.remove_oldest()
107
106
- def pop(self, key: int, default_value: Optional[Any] = None) -> Any:
108
+ def pop(self, key: Hashable, default_value: Optional[Any] = None) -> T:
109
run_on_remove = key in self.cache
110
value = self.cache.pop(key, default_value)
111
if run_on_remove: