A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 02:15 UTC
[cold start time] add envs.VLLM_COMPILE_DEPYF to guard decompile (#20790)
Signed-off-by: Boyuan Feng <boyuan@meta.com>
Boyuan Feng committed
Jul 11, 2025 at 23:06 UTC
c1c8ca57ff53a559a9bdca1ded40960f806c7505
2 files changed
+19
-3
vllm/compilation/wrapper.py
+13
-3
@@ -95,16 +95,26 @@ class TorchCompileWrapperWithCustomDispatcher:
95
self.compiled_codes.append(new_code)
96
local_cache_dir = self.vllm_config.compilation_config.local_cache_dir
97
if isinstance(local_cache_dir, str):
98
+ decompiled_file_name = ("transformed_code.py"
99
+ if envs.VLLM_COMPILE_DEPYF else
100
+ "transformed_code_README.txt")
101
+
102
decompiled_file = os.path.join(local_cache_dir,
99
- "transformed_code.py")
103
+ decompiled_file_name)
104
if not os.path.exists(decompiled_file):
105
try:
106
# usually the decompilation will succeed for most models,
107
# as we guarantee a full-graph compilation in Dynamo.
108
# but there's no 100% guarantee, since decompliation is
109
# not a reversible process.
106
- import depyf
107
- src = depyf.decompile(new_code)
110
+ if envs.VLLM_COMPILE_DEPYF:
111
+ import depyf
112
+ src = depyf.decompile(new_code)
113
+ else:
114
+ src = (
115
+ "To get a transformed_code.py file, re-run with "
116
+ "VLLM_COMPILE_DEPYF=1")
117
+
118
with open(decompiled_file, "w") as f:
119
f.write(src)
120
vllm/envs.py
+6
@@ -97,6 +97,7 @@ if TYPE_CHECKING:
97
VLLM_ENABLE_V1_MULTIPROCESSING: bool = True
98
VLLM_LOG_BATCHSIZE_INTERVAL: float = -1
99
VLLM_DISABLE_COMPILE_CACHE: bool = False
100
+ VLLM_COMPILE_DEPYF: bool = False
101
Q_SCALE_CONSTANT: int = 200
102
K_SCALE_CONSTANT: int = 200
103
V_SCALE_CONSTANT: int = 100
@@ -741,6 +742,11 @@ environment_variables: dict[str, Callable[[], Any]] = {
742
"VLLM_DISABLE_COMPILE_CACHE":
743
lambda: bool(int(os.getenv("VLLM_DISABLE_COMPILE_CACHE", "0"))),
744
745
+ # If set, vllm will decompile the torch compiled code and dump to
746
+ # transformed_code.py. This is useful for debugging.
747
+ "VLLM_COMPILE_DEPYF":
748
+ lambda: bool(int(os.getenv("VLLM_COMPILE_DEPYF", "0"))),
749
+
750
# If set, vllm will run in development mode, which will enable
751
# some additional endpoints for developing and debugging,
752
# e.g. `/reset_prefix_cache`