A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Bugfix][TPU] Set readonly=True for non-root devices (#6980)
Woosuk Kwon committed
Jul 31, 2024 at 00:19 UTC
533d1932d23917f7a29ba04c6be5c8b2676b6969
1 file changed
+4
-1
vllm/worker/tpu_worker.py
+4
-1
@@ -104,7 +104,10 @@ class TPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
104
# Use persistent cache to avoid XLA recompilation.
105
# NOTE(woosuk): This does not completely eliminate the recompilation
106
# overhead because dynamo does not cache the compiled results.
107
- xr.initialize_cache(envs.VLLM_XLA_CACHE_PATH, readonly=False)
107
+ # NOTE(woosuk): Set readonly=False only for the rank 0 process to avoid
108
+ # race conditions.
109
+ xr.initialize_cache(envs.VLLM_XLA_CACHE_PATH,
110
+ readonly=not self.is_driver_worker)
111
112
def load_model(self):
113
self.model_runner.load_model()