@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 533d1932d2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Bugfix][TPU] Set readonly=True for non-root devices (#6980)

Woosuk Kwon committed Jul 31, 2024 at 00:19 UTC 533d1932d23917f7a29ba04c6be5c8b2676b6969
1 file changed +4 -1
vllm/worker/tpu_worker.py
+4 -1
@@ -104,7 +104,10 @@ class TPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
104 # Use persistent cache to avoid XLA recompilation.
105 # NOTE(woosuk): This does not completely eliminate the recompilation
106 # overhead because dynamo does not cache the compiled results.
107 - xr.initialize_cache(envs.VLLM_XLA_CACHE_PATH, readonly=False)
107 + # NOTE(woosuk): Set readonly=False only for the rank 0 process to avoid
108 + # race conditions.
109 + xr.initialize_cache(envs.VLLM_XLA_CACHE_PATH,
110 + readonly=not self.is_driver_worker)
111
112 def load_model(self):
113 self.model_runner.load_model()