@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a1c67e6db8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Minor

Woosuk Kwon committed Feb 16, 2023 at 01:42 UTC a1c67e6db8d59c05b8785c48c1c189e6e708a2a0
1 file changed +3 -2
cacheflow/worker/cache_engine.py
+3 -2
@@ -20,7 +20,8 @@ class CacheEngine:
20 dtype: torch.dtype,
21 ) -> None:
22 if head_size % 16 != 0:
23 - raise ValueError(f'head_size ({head_size}) must be a multiple of 16.')
23 + raise ValueError(
24 + f'head_size ({head_size}) must be a multiple of 16.')
25
26 self.worker_id = worker_id
27 self.gpu_id = gpu_id
@@ -40,7 +41,7 @@ class CacheEngine:
41 self.cache_stream = torch.cuda.Stream(device=gpu_id)
42 assert self.cache_stream != torch.cuda.current_stream(device=gpu_id)
43 # Initialize the events for stream synchronization.
43 - self.events = [torch.cuda.Event() for _ in range(self.num_layers)]
44 + self.events = [torch.cuda.Event() for _ in range(num_layers)]
45
46 def get_key_block_shape(self) -> Tuple[int, int, int, int]:
47 element_size = torch.tensor([], dtype=self.dtype).element_size()