A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Bugfix][TPU] Fix KV cache size calculation (#5860)
Woosuk Kwon committed
Jun 26, 2024 at 00:58 UTC
3439c5a8e3a1cdab9bf7c4430455ace06be1f28d
1 file changed
+7
-6
vllm/worker/tpu_worker.py
+7
-6
@@ -118,14 +118,15 @@ class TPUWorker(LoraNotSupportedWorkerBase):
118
xm.wait_device_ops()
119
120
m = xm.get_memory_info(self.device)
121
- program_size = 1024 * 1024 * 1024 # 1GB
122
- free_bytes = max(m["bytes_limit"] - m["bytes_used"] - program_size, 0)
123
- kv_cache_bytes = int(free_bytes *
124
- self.cache_config.gpu_memory_utilization)
125
- kv_cache_dtype_btyes = get_dtype_size(self.cache_dtype)
121
+ total_memory_size = m["bytes_limit"]
122
+ usable_memory_size = int(total_memory_size *
123
+ self.cache_config.gpu_memory_utilization)
124
+ profiled = m["bytes_used"] # Weights + intermediate activations.
125
+ kv_cache_bytes = max(usable_memory_size - profiled, 0)
126
+ dtype_btyes = get_dtype_size(self.cache_dtype)
127
block_size = self.cache_config.block_size
128
num_tpu_blocks = (kv_cache_bytes //
128
- (kv_cache_dtype_btyes * block_size * num_layers * 2 *
129
+ (dtype_btyes * block_size * num_layers * 2 *
130
head_size * num_kv_heads))
131
num_tpu_blocks = (num_tpu_blocks // 8) * 8 # Round down to 8.
132
return num_tpu_blocks, 0