A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[TPU][Bugfix] Use XLA rank for persistent cache path (#8137)
Woosuk Kwon committed
Sep 3, 2024 at 18:35 UTC
61f4a93d1490f285b0dd3a536dd85a9f3f18ddd9
2 files changed
+3
-2
docs/source/getting_started/tpu-installation.rst
+1
-1
@@ -59,7 +59,7 @@ First, install the dependencies:
59
$ export DATE="20240828"
60
$ export TORCH_VERSION="2.5.0"
61
$ pip install https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch-${TORCH_VERSION}.dev${DATE}-cp310-cp310-linux_x86_64.whl
62
- $ pip3 install https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch_xla-${TORCH_VERSION}.dev${DATE}-cp310-cp310-linux_x86_64.whl
62
+ $ pip install https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch_xla-${TORCH_VERSION}.dev${DATE}-cp310-cp310-linux_x86_64.whl
63
64
$ # Install JAX and Pallas.
65
$ pip install torch_xla[tpu] -f https://storage.googleapis.com/libtpu-releases/index.html
vllm/worker/tpu_worker.py
+2
-1
@@ -102,8 +102,9 @@ class TPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
102
# NOTE(woosuk): Set per-rank cache path since different ranks
103
# can have slightly different XLA graphs.
104
world_size = self.parallel_config.world_size
105
+ rank = xr.global_ordinal()
106
per_rank_path = os.path.join(envs.VLLM_XLA_CACHE_PATH,
106
- f"tp{world_size}_rank{self.rank}")
107
+ f"tp{world_size}_rank{rank}")
108
xr.initialize_cache(per_rank_path, readonly=False)
109
110
def load_model(self):