A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[TPU] Set per-rank XLA cache (#7533)
Woosuk Kwon committed
Aug 14, 2024 at 14:47 UTC
951fdd66d36ffed75a55e9bfa9f2221dce4fbcdc
1 file changed
+6
-6
vllm/worker/tpu_worker.py
+6
-6
@@ -102,12 +102,12 @@ class TPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
102
# 30-40 graphs for decode. 128 is an arbitrary safe number.
103
torch._dynamo.config.cache_size_limit = 128
104
# Use persistent cache to avoid XLA recompilation.
105
- # NOTE(woosuk): This does not completely eliminate the recompilation
106
- # overhead because dynamo does not cache the compiled results.
107
- # NOTE(woosuk): Set readonly=False only for the rank 0 process to avoid
108
- # race conditions.
109
- xr.initialize_cache(envs.VLLM_XLA_CACHE_PATH,
110
- readonly=not self.is_driver_worker)
105
+ # NOTE(woosuk): Set per-rank cache path since different ranks
106
+ # can have slightly different XLA graphs.
107
+ world_size = self.parallel_config.world_size
108
+ per_rank_path = os.path.join(envs.VLLM_XLA_CACHE_PATH,
109
+ f"tp{world_size}_rank{self.rank}")
110
+ xr.initialize_cache(per_rank_path, readonly=False)
111
112
def load_model(self):
113
self.model_runner.load_model()