A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 02:00 UTC
[Perf] Disable inductor runtime asserts by default for serving perfor… (#37485)
Signed-off-by: tianrengao <terrygao87@gmail.com> Co-authored-by: Tianren Gao <tianren@fb.com>
Terry Gao committed
Mar 24, 2026 at 16:37 UTC
82580b10acaac3b129d84f17d3defbfbb2eeb161
3 files changed
+95
docs/design/debug_vllm_compile.md
+20
@@ -233,6 +233,26 @@ that may call 1+ triton kernels. On rare (but unfortunate) occasions, it may
233
produce an incorrect triton kernel. This may manifest as silent incorrectness,
234
CUDA illegal memory accesses, or loud errors.
235
236
+### Inductor runtime assertions
237
+
238
+By default (on torch < 2.12), vLLM disables Inductor's runtime assertions
239
+(`assert_size_stride`, `assert_alignment`) to avoid ~2ms overhead per forward
240
+pass on large models. Setting `VLLM_LOGGING_LEVEL=DEBUG` automatically
241
+re-enables them so debugging sessions get full shape/stride validation:
242
+
243
+```sh
244
+VLLM_LOGGING_LEVEL=DEBUG vllm serve <model>
245
+```
246
+
247
+You can also override them explicitly via `--compilation-config`:
248
+
249
+```sh
250
+vllm serve <model> -cc.inductor_compile_config='{"size_asserts": true, "alignment_asserts": true, "scalar_asserts": true}'
251
+```
252
+
253
+On torch >= 2.12, PyTorch uses an efficient assert-once strategy and these
254
+flags are no longer suppressed by vLLM.
255
+
256
To debug if TorchInductor is at fault, you can disable it by passing `backend='eager'`
257
to the compilation config:
258
tests/compile/test_config.py
+56
@@ -5,6 +5,7 @@ from contextlib import nullcontext
5
from unittest.mock import MagicMock, patch
6
7
import pytest
8
+import torch
9
from pydantic import ValidationError
10
11
from vllm.compilation.counter import compilation_counter
@@ -612,3 +613,58 @@ def test_adjust_cudagraph_sizes_for_mamba_cache(
613
# Invariant: last element == max_cudagraph_capture_size
614
if expected_sizes:
615
assert config.cudagraph_capture_sizes[-1] == config.max_cudagraph_capture_size
616
+
617
+
618
+def test_inductor_asserts_default_disabled(monkeypatch):
619
+ """Test that inductor runtime asserts are disabled by default
620
+ (INFO logging level) on torch < 2.12."""
621
+ monkeypatch.setenv("VLLM_LOGGING_LEVEL", "INFO")
622
+
623
+ import importlib
624
+
625
+ import vllm.envs
626
+
627
+ importlib.reload(vllm.envs)
628
+
629
+ config = CompilationConfig()
630
+ if not _is_torch_equal_or_newer(torch.__version__, "2.12.0.dev"):
631
+ assert config.inductor_compile_config.get("size_asserts") is False
632
+ assert config.inductor_compile_config.get("alignment_asserts") is False
633
+ assert config.inductor_compile_config.get("scalar_asserts") is False
634
+
635
+
636
+def test_inductor_asserts_enabled_in_debug(monkeypatch):
637
+ """Test that VLLM_LOGGING_LEVEL=DEBUG enables inductor runtime asserts
638
+ on torch < 2.12."""
639
+ monkeypatch.setenv("VLLM_LOGGING_LEVEL", "DEBUG")
640
+
641
+ import importlib
642
+
643
+ import vllm.envs
644
+
645
+ importlib.reload(vllm.envs)
646
+
647
+ config = CompilationConfig()
648
+ if not _is_torch_equal_or_newer(torch.__version__, "2.12.0.dev"):
649
+ assert config.inductor_compile_config.get("size_asserts") is True
650
+ assert config.inductor_compile_config.get("alignment_asserts") is True
651
+ assert config.inductor_compile_config.get("scalar_asserts") is True
652
+
653
+
654
+def test_inductor_asserts_user_override(monkeypatch):
655
+ """Test that explicit inductor_compile_config overrides the
656
+ debug-logging default."""
657
+ monkeypatch.setenv("VLLM_LOGGING_LEVEL", "INFO")
658
+
659
+ import importlib
660
+
661
+ import vllm.envs
662
+
663
+ importlib.reload(vllm.envs)
664
+
665
+ config = CompilationConfig(
666
+ inductor_compile_config={"size_asserts": True},
667
+ )
668
+ assert config.inductor_compile_config.get("size_asserts") is True
669
+ if not _is_torch_equal_or_newer(torch.__version__, "2.12.0.dev"):
670
+ assert config.inductor_compile_config.get("alignment_asserts") is False
vllm/config/compilation.py
+19
@@ -858,6 +858,25 @@ class CompilationConfig:
858
if KEY not in self.inductor_compile_config:
859
self.inductor_compile_config[KEY] = False
860
861
+ # Tie inductor runtime assertions to debug logging mode.
862
+ # These assertions add ~2ms overhead per forward pass on large
863
+ # models (e.g., DeepSeek-R1 671B: ~340 assert_size_stride + ~60
864
+ # assert_alignment calls per forward). PyTorch >= 2.12 has a
865
+ # native fix (assert-once), so we only apply this workaround on
866
+ # older versions. On torch < 2.12, enable asserts only when
867
+ # VLLM_LOGGING_LEVEL=DEBUG. Users can still override explicitly
868
+ # via --compilation-config '{"inductor_compile_config":
869
+ # {"size_asserts": true, ...}}'.
870
+ # See: https://github.com/pytorch/pytorch/issues/177719
871
+ if not is_torch_equal_or_newer("2.12.0.dev"):
872
+ enable_asserts = envs.VLLM_LOGGING_LEVEL == "DEBUG"
873
+ for key in (
874
+ "size_asserts",
875
+ "alignment_asserts",
876
+ "scalar_asserts",
877
+ ):
878
+ self.inductor_compile_config.setdefault(key, enable_asserts)
879
+
880
for k, v in self.inductor_passes.items():
881
if not isinstance(v, str):
882
assert callable(v), f"pass {k} should be callable or a qualified name"