@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9e68a6827e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Fix return type error

Woosuk Kwon committed Feb 16, 2023 at 01:33 UTC 9e68a6827ef7e08c71b9c4a92cf3c2be7b60fc84
1 file changed +2 -2
cacheflow/worker/cache_engine.py
+2 -2
@@ -42,7 +42,7 @@ class CacheEngine:
42 # Initialize the events for stream synchronization.
43 self.events = [torch.cuda.Event() for _ in range(self.num_layers)]
44
45 - def get_key_block_shape(self) -> Tuple[int, int, int, int, int]:
45 + def get_key_block_shape(self) -> Tuple[int, int, int, int]:
46 element_size = torch.tensor([], dtype=self.dtype).element_size()
47 x = 16 // element_size
48 return (
@@ -52,7 +52,7 @@ class CacheEngine:
52 x,
53 )
54
55 - def get_value_block_shape(self) -> Tuple[int, int, int, int]:
55 + def get_value_block_shape(self) -> Tuple[int, int, int]:
56 return (
57 self.num_heads,
58 self.block_size,