A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 14:45 UTC
Disable remote caching when calling compile_fx (#16611)
Signed-off-by: rzou <zou3519@gmail.com>
Richard Zou committed
Apr 16, 2025 at 01:18 UTC
966c742ed2955db9515509a3351248e5c7b5a3a2
1 file changed
+13
vllm/compilation/compiler_interface.py
+13
@@ -290,6 +290,19 @@ class InductorAdaptor(CompilerInterface):
290
# Dynamo metrics context, see method for more details.
291
stack.enter_context(self.metrics_context())
292
293
+ # Disable remote caching. When these are on, on remote cache-hit,
294
+ # the monkey-patched functions never actually get called.
295
+ # vLLM today assumes and requires the monkey-patched functions to
296
+ # get hit.
297
+ # TODO(zou3519): we're going to replace this all with
298
+ # standalone_compile sometime.
299
+ if is_torch_equal_or_newer("2.6"):
300
+ stack.enter_context(
301
+ torch._inductor.config.patch(fx_graph_remote_cache=False))
302
+ stack.enter_context(
303
+ torch._functorch.config.patch(
304
+ enable_remote_autograd_cache=False))
305
+
306
compiled_graph = compile_fx(
307
graph,
308
example_inputs,