@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3439c5a8e3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Bugfix][TPU] Fix KV cache size calculation (#5860)

Woosuk Kwon committed Jun 26, 2024 at 00:58 UTC 3439c5a8e3a1cdab9bf7c4430455ace06be1f28d
1 file changed +7 -6
vllm/worker/tpu_worker.py
+7 -6
@@ -118,14 +118,15 @@ class TPUWorker(LoraNotSupportedWorkerBase):
118 xm.wait_device_ops()
119
120 m = xm.get_memory_info(self.device)
121 - program_size = 1024 * 1024 * 1024 # 1GB
122 - free_bytes = max(m["bytes_limit"] - m["bytes_used"] - program_size, 0)
123 - kv_cache_bytes = int(free_bytes *
124 - self.cache_config.gpu_memory_utilization)
125 - kv_cache_dtype_btyes = get_dtype_size(self.cache_dtype)
121 + total_memory_size = m["bytes_limit"]
122 + usable_memory_size = int(total_memory_size *
123 + self.cache_config.gpu_memory_utilization)
124 + profiled = m["bytes_used"] # Weights + intermediate activations.
125 + kv_cache_bytes = max(usable_memory_size - profiled, 0)
126 + dtype_btyes = get_dtype_size(self.cache_dtype)
127 block_size = self.cache_config.block_size
128 num_tpu_blocks = (kv_cache_bytes //
128 - (kv_cache_dtype_btyes * block_size * num_layers * 2 *
129 + (dtype_btyes * block_size * num_layers * 2 *
130 head_size * num_kv_heads))
131 num_tpu_blocks = (num_tpu_blocks // 8) * 8 # Round down to 8.
132 return num_tpu_blocks, 0