A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 14:00 UTC
[Bugfix] Add monkeypatch to prevent race condition from writing (#35420)
Signed-off-by: Lucas Kabela <lucaskabela@meta.com>
Lucas Kabela committed
Feb 27, 2026 at 11:51 UTC
234a65b781d9dc51d28aebb208096baa8fe0458e
1 file changed
+43
vllm/compilation/compiler_interface.py
+43
@@ -184,6 +184,47 @@ def is_compile_cache_enabled(
184
)
185
186
187
+def _patch_standalone_compile_atomic_save() -> None:
188
+ """Backport of pytorch/pytorch#162432 for torch < 2.10.0.
189
+
190
+ Patches CompiledArtifact.save() to use write_atomic for binary format,
191
+ preventing corrupt cache files when multiple processes compile
192
+ concurrently.
193
+ """
194
+ from torch._inductor.codecache import write_atomic
195
+ from torch._inductor.standalone_compile import CompiledArtifact as cls
196
+
197
+ if getattr(cls.save, "_vllm_patched", False):
198
+ return
199
+
200
+ original_save = cls.save
201
+
202
+ def _save(
203
+ self: Any, *, path: str, format: Literal["binary", "unpacked"] = "binary"
204
+ ) -> None:
205
+ if format != "binary":
206
+ return original_save(self, path=path, format=format)
207
+ from torch._dynamo.utils import dynamo_timed
208
+ from torch._inductor.codecache import torch_key
209
+ from torch.utils._appending_byte_serializer import BytesWriter
210
+
211
+ with dynamo_timed("CompiledArtifact.save"):
212
+ assert self._artifacts is not None
213
+ artifact_bytes, cache_info = self._artifacts
214
+ assert len(cache_info.aot_autograd_artifacts) == 1, cache_info
215
+ key = cache_info.aot_autograd_artifacts[0]
216
+ assert not os.path.isdir(path)
217
+ writer = BytesWriter()
218
+ writer.write_bytes(torch_key())
219
+ writer.write_str(key)
220
+ writer.write_bytes(artifact_bytes)
221
+ write_atomic(path, writer.to_bytes())
222
+
223
+ _save._vllm_patched = True # type: ignore[attr-defined]
224
+ cls.save = _save # type: ignore[assignment]
225
+ logger.debug("Patched %s.save for atomic writes (torch < 2.10)", cls.__name__)
226
+
227
+
228
class InductorStandaloneAdaptor(CompilerInterface):
229
"""
230
The adaptor for the Inductor compiler.
@@ -197,6 +238,8 @@ class InductorStandaloneAdaptor(CompilerInterface):
238
name = "inductor_standalone"
239
240
def __init__(self, save_format: Literal["binary", "unpacked"]) -> None:
241
+ if not is_torch_equal_or_newer("2.10.0"):
242
+ _patch_standalone_compile_atomic_save()
243
self.save_format = save_format
244
245
def compute_hash(self, vllm_config: VllmConfig) -> str: