@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 211fe91aa8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[TPU] Correctly profile peak memory usage & Upgrade PyTorch XLA (#9438)

Woosuk Kwon committed Oct 30, 2024 at 02:41 UTC 211fe91aa88730c04df439298d8103a587302493
3 files changed +11 -10
Dockerfile.tpu
+1 -1
@@ -1,4 +1,4 @@
1 -ARG NIGHTLY_DATE="20240828"
1 +ARG NIGHTLY_DATE="20241017"
2 ARG BASE_IMAGE="us-central1-docker.pkg.dev/tpu-pytorch-releases/docker/xla:nightly_3.10_tpuvm_$NIGHTLY_DATE"
3
4 FROM $BASE_IMAGE
docs/source/getting_started/tpu-installation.rst
+2 -2
@@ -56,8 +56,8 @@ First, install the dependencies:
56 $ pip uninstall torch torch-xla -y
57
58 $ # Install PyTorch and PyTorch XLA.
59 - $ export DATE="20240828"
60 - $ export TORCH_VERSION="2.5.0"
59 + $ export DATE="20241017"
60 + $ export TORCH_VERSION="2.6.0"
61 $ pip install https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch-${TORCH_VERSION}.dev${DATE}-cp310-cp310-linux_x86_64.whl
62 $ pip install https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch_xla-${TORCH_VERSION}.dev${DATE}-cp310-cp310-linux_x86_64.whl
63
vllm/worker/tpu_worker.py
+8 -7
@@ -133,18 +133,19 @@ class TPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
133 # Synchronize before measuring the memory usage.
134 xm.wait_device_ops()
135
136 - dtype_btyes = get_dtype_size(self.cache_dtype)
137 - block_size = self.cache_config.block_size
138 - block_size_bytes = (dtype_btyes * block_size * num_layers * 2 *
139 - head_size * num_kv_heads)
140 -
141 - # Calculate the TPU KV cache size based on profiling.
136 + # Get the maximum amount of memory used by the model weights and
137 + # intermediate activations.
138 m = xm.get_memory_info(self.device)
139 total_memory_size = m["bytes_limit"]
140 + profiled = m["peak_bytes_used"] # Weights + intermediate activations.
141 +
142 + # Calculate the TPU KV cache size based on profiling.
143 usable_memory_size = int(total_memory_size *
144 self.cache_config.gpu_memory_utilization)
146 - profiled = m["bytes_used"] # Weights + intermediate activations.
145 tpu_kv_cache_bytes = max(usable_memory_size - profiled, 0)
146 + dtype_btyes = get_dtype_size(self.cache_dtype)
147 + block_size_bytes = (dtype_btyes * self.cache_config.block_size *
148 + num_layers * 2 * head_size * num_kv_heads)
149 num_tpu_blocks = tpu_kv_cache_bytes // block_size_bytes
150 num_tpu_blocks = (num_tpu_blocks // 8) * 8 # Round down to 8.
151