A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 09:30 UTC
[Performance] Enable Triton autotuning disk cache by default (#37188)
Signed-off-by: Artem Perevedentsev <aperevedents@nvidia.com>
Artem Perevedentsev committed
Mar 19, 2026 at 23:36 UTC
b55156eae9aa586b8fbeb752ecb369179442c521
1 file changed
+8
vllm/env_override.py
+8
@@ -105,6 +105,14 @@ os.environ["TORCHINDUCTOR_COMPILE_THREADS"] = "1"
105
# see https://github.com/vllm-project/vllm/issues/10619
106
torch._inductor.config.compile_threads = 1
107
108
+# Enable Triton autotuning result caching to disk by default.
109
+# Without this, Triton re-runs autotuning on every process restart,
110
+# adding significant latency to the first inference request.
111
+# This writes autotuning results to TRITON_CACHE_DIR.
112
+# It can still be overridden by setting TRITON_CACHE_AUTOTUNING=0
113
+# in the environment.
114
+os.environ.setdefault("TRITON_CACHE_AUTOTUNING", "1")
115
+
116
# ===================================================
117
# torch 2.9 Inductor PythonWrapperCodegen monkeypatch
118
# ===================================================