A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 06:00 UTC
use base version for version comparison (#19587)
Signed-off-by: Boyuan Feng <boyuan@meta.com>
Boyuan Feng committed
Jun 13, 2025 at 00:09 UTC
ce688ad46eb02c49a049cbefd9e7424dbc09fd53
2 files changed
+3
-3
vllm/compilation/backends.py
+1
-1
@@ -31,7 +31,7 @@ logger = init_logger(__name__)
31
def make_compiler(compilation_config: CompilationConfig) -> CompilerInterface:
32
if compilation_config.use_inductor:
33
if envs.VLLM_USE_STANDALONE_COMPILE and is_torch_equal_or_newer(
34
- "2.8.0"):
34
+ "2.8.0a"):
35
logger.debug("Using InductorStandaloneAdaptor")
36
return InductorStandaloneAdaptor()
37
else:
vllm/model_executor/layers/quantization/torchao.py
+2
-2
@@ -44,14 +44,14 @@ class TorchAOConfig(QuantizationConfig):
44
"""
45
# TorchAO quantization relies on tensor subclasses. In order,
46
# to enable proper caching this needs standalone compile
47
- if is_torch_equal_or_newer("2.8.0"):
47
+ if is_torch_equal_or_newer("2.8.0a"):
48
os.environ["VLLM_TEST_STANDALONE_COMPILE"] = "1"
49
logger.info(
50
"Using TorchAO: Setting VLLM_TEST_STANDALONE_COMPILE=1")
51
52
# TODO: remove after the torch dependency is updated to 2.8
53
if is_torch_equal_or_newer(
54
- "2.7.0") and not is_torch_equal_or_newer("2.8.0"):
54
+ "2.7.0") and not is_torch_equal_or_newer("2.8.0a"):
55
os.environ["VLLM_DISABLE_COMPILE_CACHE"] = "1"
56
logger.info("Using TorchAO: Setting VLLM_DISABLE_COMPILE_CACHE=1")
57
"""