A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 02:00 UTC
[BugFix] fix VLLM_USE_STANDALONE_COMPILE=0 (#38015)
Signed-off-by: Richard Zou <zou3519@gmail.com>
Richard Zou committed
Mar 24, 2026 at 15:08 UTC
89f572dbc0bf18efaaec52b95119bf4c01330dfe
2 files changed
+48
tests/compile/test_aot_compile.py
+31
@@ -441,6 +441,37 @@ def test_partition_wrapper_applied_on_aot_load(
441
)
442
443
444
+@create_new_process_for_each_test("spawn")
445
+def test_standalone_compile_correctness():
446
+ """Outputs must match regardless of VLLM_USE_STANDALONE_COMPILE."""
447
+ import json
448
+
449
+ from ..utils import compare_two_settings
450
+
451
+ compilation_config = json.dumps(
452
+ {
453
+ "mode": CompilationMode.VLLM_COMPILE,
454
+ }
455
+ )
456
+
457
+ common_args = [
458
+ "--dtype",
459
+ "float16",
460
+ "--max-model-len",
461
+ "256",
462
+ "--compilation_config",
463
+ compilation_config,
464
+ ]
465
+
466
+ compare_two_settings(
467
+ "facebook/opt-125m",
468
+ common_args,
469
+ common_args,
470
+ env1={"VLLM_USE_STANDALONE_COMPILE": "1"},
471
+ env2={"VLLM_USE_STANDALONE_COMPILE": "0"},
472
+ )
473
+
474
+
475
@pytest.mark.skipif(not is_torch_equal_or_newer("2.10.0"), reason="requires torch 2.10")
476
@create_new_process_for_each_test("spawn")
477
def test_gpt2_cache_hit(monkeypatch: pytest.MonkeyPatch):
vllm/compilation/compiler_interface.py
+17
@@ -632,6 +632,23 @@ class InductorAdaptor(CompilerInterface):
632
)
633
stack.enter_context(_patch_constrain_to_fx_strides())
634
635
+ # Clear the tracing context before calling compile_fx.
636
+ # vLLM calls compile_fx from within a PiecewiseCompileInterpreter
637
+ # that runs under Dynamo's tracing context. The tracing context
638
+ # has a FakeTensorMode from Dynamo, but the example inputs for
639
+ # this subgraph have fake tensors from a different FakeTensorMode.
640
+ # compile_fx's _compile_fx_main calls detect_fake_mode() which
641
+ # asserts all FakeTensorModes match, causing a crash.
642
+ # Clearing the tracing context lets compile_fx create its own.
643
+ saved_tracing_context = torch._guards.TracingContext.try_get()
644
+ if saved_tracing_context is not None:
645
+ torch._guards._TLS.tracing_context = None
646
+
647
+ def _restore_tracing_context():
648
+ torch._guards._TLS.tracing_context = saved_tracing_context
649
+
650
+ stack.callback(_restore_tracing_context)
651
+
652
compiled_graph = compile_fx(
653
graph,
654
example_inputs,