A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Fix return type error
Woosuk Kwon committed
Feb 16, 2023 at 01:33 UTC
9e68a6827ef7e08c71b9c4a92cf3c2be7b60fc84
1 file changed
+2
-2
cacheflow/worker/cache_engine.py
+2
-2
@@ -42,7 +42,7 @@ class CacheEngine:
42
# Initialize the events for stream synchronization.
43
self.events = [torch.cuda.Event() for _ in range(self.num_layers)]
44
45
- def get_key_block_shape(self) -> Tuple[int, int, int, int, int]:
45
+ def get_key_block_shape(self) -> Tuple[int, int, int, int]:
46
element_size = torch.tensor([], dtype=self.dtype).element_size()
47
x = 16 // element_size
48
return (
@@ -52,7 +52,7 @@ class CacheEngine:
52
x,
53
)
54
55
- def get_value_block_shape(self) -> Tuple[int, int, int, int]:
55
+ def get_value_block_shape(self) -> Tuple[int, int, int]:
56
return (
57
self.num_heads,
58
self.block_size,