@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 951fdd66d3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[TPU] Set per-rank XLA cache (#7533)

Woosuk Kwon committed Aug 14, 2024 at 14:47 UTC 951fdd66d36ffed75a55e9bfa9f2221dce4fbcdc
1 file changed +6 -6
vllm/worker/tpu_worker.py
+6 -6
@@ -102,12 +102,12 @@ class TPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
102 # 30-40 graphs for decode. 128 is an arbitrary safe number.
103 torch._dynamo.config.cache_size_limit = 128
104 # Use persistent cache to avoid XLA recompilation.
105 - # NOTE(woosuk): This does not completely eliminate the recompilation
106 - # overhead because dynamo does not cache the compiled results.
107 - # NOTE(woosuk): Set readonly=False only for the rank 0 process to avoid
108 - # race conditions.
109 - xr.initialize_cache(envs.VLLM_XLA_CACHE_PATH,
110 - readonly=not self.is_driver_worker)
105 + # NOTE(woosuk): Set per-rank cache path since different ranks
106 + # can have slightly different XLA graphs.
107 + world_size = self.parallel_config.world_size
108 + per_rank_path = os.path.join(envs.VLLM_XLA_CACHE_PATH,
109 + f"tp{world_size}_rank{self.rank}")
110 + xr.initialize_cache(per_rank_path, readonly=False)
111
112 def load_model(self):
113 self.model_runner.load_model()