A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Minor
Woosuk Kwon committed
Feb 16, 2023 at 01:42 UTC
a1c67e6db8d59c05b8785c48c1c189e6e708a2a0
1 file changed
+3
-2
cacheflow/worker/cache_engine.py
+3
-2
@@ -20,7 +20,8 @@ class CacheEngine:
20
dtype: torch.dtype,
21
) -> None:
22
if head_size % 16 != 0:
23
- raise ValueError(f'head_size ({head_size}) must be a multiple of 16.')
23
+ raise ValueError(
24
+ f'head_size ({head_size}) must be a multiple of 16.')
25
26
self.worker_id = worker_id
27
self.gpu_id = gpu_id
@@ -40,7 +41,7 @@ class CacheEngine:
41
self.cache_stream = torch.cuda.Stream(device=gpu_id)
42
assert self.cache_stream != torch.cuda.current_stream(device=gpu_id)
43
# Initialize the events for stream synchronization.
43
- self.events = [torch.cuda.Event() for _ in range(self.num_layers)]
44
+ self.events = [torch.cuda.Event() for _ in range(num_layers)]
45
46
def get_key_block_shape(self) -> Tuple[int, int, int, int]:
47
element_size = torch.tensor([], dtype=self.dtype).element_size()