@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4ad521d8b5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[Core] Add generic typing to `LRUCache` (#3511)

Nick Hill committed Mar 20, 2024 at 00:36 UTC 4ad521d8b51145a55c1be6b8e451f76423cc2d87
4 files changed +29 -22
vllm/lora/models.py
+3 -3
@@ -4,7 +4,7 @@ import logging
4 import math
5 import os
6 import re
7 -from typing import (Any, Callable, Dict, Hashable, List, Optional, Tuple, Type)
7 +from typing import (Callable, Dict, Hashable, List, Optional, Tuple, Type)
8
9 import safetensors.torch
10 import torch
@@ -535,14 +535,14 @@ class LoRAModelManager:
535 replacement_loras)
536
537
538 -class LoRALRUCache(LRUCache):
538 +class LoRALRUCache(LRUCache[LoRAModel]):
539
540 def __init__(self, capacity: int, deactivate_lora_fn: Callable[[Hashable],
541 None]):
542 super().__init__(capacity)
543 self.deactivate_lora_fn = deactivate_lora_fn
544
545 - def _on_remove(self, key: Hashable, value: Any):
545 + def _on_remove(self, key: Hashable, value: LoRAModel):
546 logger.debug(f"Removing LoRA. int id: {key}")
547 self.deactivate_lora_fn(key)
548 return super()._on_remove(key, value)
vllm/transformers_utils/tokenizer_group/base_tokenizer_group.py
+13 -6
@@ -22,27 +22,34 @@ class BaseTokenizerGroup(ABC):
22 pass
23
24 @abstractmethod
25 - def encode(self, prompt: str, request_id: Optional[str],
26 - lora_request: Optional[LoRARequest]) -> List[int]:
25 + def encode(self,
26 + prompt: str,
27 + request_id: Optional[str] = None,
28 + lora_request: Optional[LoRARequest] = None) -> List[int]:
29 """Encode a prompt using the tokenizer group."""
30 pass
31
32 @abstractmethod
31 - async def encode_async(self, prompt: str, request_id: Optional[str],
32 - lora_request: Optional[LoRARequest]) -> List[int]:
33 + async def encode_async(
34 + self,
35 + prompt: str,
36 + request_id: Optional[str] = None,
37 + lora_request: Optional[LoRARequest] = None) -> List[int]:
38 """Encode a prompt using the tokenizer group."""
39 pass
40
41 @abstractmethod
42 def get_lora_tokenizer(
43 self,
39 - lora_request: Optional[LoRARequest]) -> "PreTrainedTokenizer":
44 + lora_request: Optional[LoRARequest] = None
45 + ) -> "PreTrainedTokenizer":
46 """Get a tokenizer for a LoRA request."""
47 pass
48
49 @abstractmethod
50 async def get_lora_tokenizer_async(
51 self,
46 - lora_request: Optional[LoRARequest]) -> "PreTrainedTokenizer":
52 + lora_request: Optional[LoRARequest] = None
53 + ) -> "PreTrainedTokenizer":
54 """Get a tokenizer for a LoRA request."""
55 pass
vllm/transformers_utils/tokenizer_group/tokenizer_group.py
+2 -4
@@ -21,10 +21,8 @@ class TokenizerGroup(BaseTokenizerGroup):
21 self.enable_lora = enable_lora
22 self.max_input_length = max_input_length
23 self.tokenizer = get_tokenizer(self.tokenizer_id, **tokenizer_config)
24 - if enable_lora:
25 - self.lora_tokenizers = LRUCache(capacity=max_num_seqs)
26 - else:
27 - self.lora_tokenizers = None
24 + self.lora_tokenizers = LRUCache[PreTrainedTokenizer](
25 + capacity=max_num_seqs) if enable_lora else None
26
27 def ping(self) -> bool:
28 """Check if the tokenizer group is alive."""
vllm/utils.py
+11 -9
@@ -5,7 +5,7 @@ import subprocess
5 import uuid
6 import gc
7 from platform import uname
8 -from typing import List, Tuple, Union
8 +from typing import List, Tuple, Union, Generic
9 from packaging.version import parse, Version
10
11 import psutil
@@ -53,10 +53,10 @@ class Counter:
53 self.counter = 0
54
55
56 -class LRUCache:
56 +class LRUCache(Generic[T]):
57
58 def __init__(self, capacity: int):
59 - self.cache = OrderedDict()
59 + self.cache = OrderedDict[Hashable, T]()
60 self.capacity = capacity
61
62 def __contains__(self, key: Hashable) -> bool:
@@ -65,10 +65,10 @@ class LRUCache:
65 def __len__(self) -> int:
66 return len(self.cache)
67
68 - def __getitem__(self, key: Hashable) -> Any:
68 + def __getitem__(self, key: Hashable) -> T:
69 return self.get(key)
70
71 - def __setitem__(self, key: Hashable, value: Any) -> None:
71 + def __setitem__(self, key: Hashable, value: T) -> None:
72 self.put(key, value)
73
74 def __delitem__(self, key: Hashable) -> None:
@@ -77,7 +77,9 @@ class LRUCache:
77 def touch(self, key: Hashable) -> None:
78 self.cache.move_to_end(key)
79
80 - def get(self, key: Hashable, default_value: Optional[Any] = None) -> int:
80 + def get(self,
81 + key: Hashable,
82 + default_value: Optional[T] = None) -> Optional[T]:
83 if key in self.cache:
84 value = self.cache[key]
85 self.cache.move_to_end(key)
@@ -85,12 +87,12 @@ class LRUCache:
87 value = default_value
88 return value
89
88 - def put(self, key: Hashable, value: Any) -> None:
90 + def put(self, key: Hashable, value: T) -> None:
91 self.cache[key] = value
92 self.cache.move_to_end(key)
93 self._remove_old_if_needed()
94
93 - def _on_remove(self, key: Hashable, value: Any):
95 + def _on_remove(self, key: Hashable, value: T):
96 pass
97
98 def remove_oldest(self):
@@ -103,7 +105,7 @@ class LRUCache:
105 while len(self.cache) > self.capacity:
106 self.remove_oldest()
107
106 - def pop(self, key: int, default_value: Optional[Any] = None) -> Any:
108 + def pop(self, key: Hashable, default_value: Optional[Any] = None) -> T:
109 run_on_remove = key in self.cache
110 value = self.cache.pop(key, default_value)
111 if run_on_remove: