A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 07:15 UTC
Use standalone_compile by default in torch >= 2.8.0 (#18846)
Signed-off-by: rzou <zou3519@gmail.com>
Richard Zou committed
May 29, 2025 at 18:41 UTC
a521ef06e5eb18a34d665282fa38c4768a855bb8
3 files changed
+12
-9
vllm/compilation/backends.py
+3
-2
@@ -16,7 +16,7 @@ import vllm.envs as envs
16
from vllm.config import CompilationConfig, VllmConfig
17
from vllm.logger import init_logger
18
from vllm.platforms import current_platform
19
-from vllm.utils import resolve_obj_by_qualname
19
+from vllm.utils import is_torch_equal_or_newer, resolve_obj_by_qualname
20
21
from .compiler_interface import (CompilerInterface, EagerAdaptor,
22
InductorAdaptor, InductorStandaloneAdaptor)
@@ -29,7 +29,8 @@ logger = init_logger(__name__)
29
30
def make_compiler(compilation_config: CompilationConfig) -> CompilerInterface:
31
if compilation_config.use_inductor:
32
- if envs.VLLM_TEST_STANDALONE_COMPILE:
32
+ if envs.VLLM_USE_STANDALONE_COMPILE and is_torch_equal_or_newer(
33
+ "2.8.0"):
34
logger.info("Using InductorStandaloneAdaptor")
35
return InductorStandaloneAdaptor()
36
else:
vllm/compilation/compiler_interface.py
+1
-1
@@ -155,7 +155,7 @@ class InductorStandaloneAdaptor(CompilerInterface):
155
This is not on by default yet, but we plan to turn it on by default for
156
PyTorch 2.8.
157
158
- Use VLLM_TEST_STANDALONE_COMPILE to toggle this on or off.
158
+ Use VLLM_USE_STANDALONE_COMPILE to toggle this on or off.
159
"""
160
name = "inductor_standalone"
161
vllm/envs.py
+8
-6
@@ -143,10 +143,10 @@ def maybe_convert_int(value: Optional[str]) -> Optional[int]:
143
144
def get_vllm_port() -> Optional[int]:
145
"""Get the port from VLLM_PORT environment variable.
146
-
146
+
147
Returns:
148
The port number as an integer if VLLM_PORT is set, None otherwise.
149
-
149
+
150
Raises:
151
ValueError: If VLLM_PORT is a URI, suggest k8s service discovery issue.
152
"""
@@ -308,9 +308,11 @@ environment_variables: dict[str, Callable[[], Any]] = {
308
lambda: bool(
309
os.environ.get("VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE", "1") != "0"),
310
311
- # Internal flag to enable/disable Inductor standalone compile
312
- "VLLM_TEST_STANDALONE_COMPILE":
313
- lambda: os.environ.get("VLLM_TEST_STANDALONE_COMPILE", "0") != "0",
311
+ # Feature flag to enable/disable Inductor standalone compile.
312
+ # In torch <= 2.7 we ignore this flag; in torch >= 2.8 this is
313
+ # enabled by default.
314
+ "VLLM_USE_STANDALONE_COMPILE":
315
+ lambda: os.environ.get("VLLM_USE_STANDALONE_COMPILE", "1") == "1",
316
317
# local rank of the process in the distributed setting, used to determine
318
# the GPU device id
@@ -892,7 +894,7 @@ def compute_hash() -> str:
894
"VLLM_USE_TRITON_AWQ",
895
"VLLM_DP_RANK",
896
"VLLM_DP_SIZE",
895
- "VLLM_TEST_STANDALONE_COMPILE",
897
+ "VLLM_USE_STANDALONE_COMPILE",
898
]
899
for key in environment_variables_to_hash:
900
if key in environment_variables: