A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Add warning
Woosuk Kwon committed
Feb 16, 2023 at 01:28 UTC
8edcabc737c867cf237289134583c2bbc852f11e
1 file changed
+3
cacheflow/worker/cache_engine.py
+3
@@ -19,6 +19,9 @@ class CacheEngine:
19
num_cpu_blocks: int,
20
dtype: torch.dtype,
21
) -> None:
22
+ if head_size % 16 != 0:
23
+ raise ValueError(f'head_size ({head_size}) must be a multiple of 16.')
24
+
25
self.worker_id = worker_id
26
self.gpu_id = gpu_id
27
self.num_layers = num_layers