@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cac4c10ef0

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 13, 20:45 UTC

[BUG] Make 'binary' default option for saving torch compile artifacts when using standalone_compile (#27616)

Signed-off-by: ahao-anyscale <ahao@anyscale.com>

ahao-anyscale committed Nov 3, 2025 at 08:13 UTC cac4c10ef0e3280f045bff32cbb05e9a56e41b1b

5 files changed +43 -5

docs/design/torch_compile.md

@@ -27,6 +27,8 @@ With all these factors taken into consideration, usually we can guarantee that t

28 A unique aspect of vLLM's `torch.compile` integration, is that we guarantee all the compilation finishes before we serve any requests. No requests will trigger new compilations. Otherwise, the engine would be blocked on that request, and the response time will have unexpected spikes.

30 +By default, the cache saves compiled artifacts as binary files. If you would like to interact with the generated code for debugging purposes, set the field `compile_cache_save_format=unpacked` in the compilation config, or omit this and set the env variable `VLLM_COMPILE_CACHE_SAVE_FORMAT=unpacked`.

31 +

32 ## Python Code Compilation

34 In the very verbose logs, we can see:

vllm/compilation/backends.py

+3 -1

@@ -51,7 +51,9 @@ def make_compiler(compilation_config: CompilationConfig) -> CompilerInterface:

51 and hasattr(torch._inductor, "standalone_compile")

52 ):

53 logger.debug("Using InductorStandaloneAdaptor")

54 - return InductorStandaloneAdaptor()

54 + return InductorStandaloneAdaptor(

55 + compilation_config.compile_cache_save_format

56 + )

57 else:

58 logger.debug("Using InductorAdaptor")

59 return InductorAdaptor()

vllm/compilation/compiler_interface.py

+6 -3

@@ -6,7 +6,7 @@ import hashlib

6 import os

7 from collections.abc import Callable

8 from contextlib import ExitStack

9 -from typing import Any

9 +from typing import Any, Literal

10 from unittest.mock import patch

12 import torch

@@ -175,6 +175,9 @@ class InductorStandaloneAdaptor(CompilerInterface):

175

176 name = "inductor_standalone"

177

178 + def __init__(self, save_format: Literal["binary", "unpacked"]):

179 + self.save_format = save_format

180 +

181 def compute_hash(self, vllm_config: VllmConfig) -> str:

182 factors = get_inductor_factors()

183 hash_str = hashlib.md5(

@@ -220,7 +223,7 @@ class InductorStandaloneAdaptor(CompilerInterface):

223 assert key is not None

224 path = os.path.join(self.cache_dir, key)

225 if not envs.VLLM_DISABLE_COMPILE_CACHE:

223 - compiled_graph.save(path=path, format="unpacked")

226 + compiled_graph.save(path=path, format=self.save_format)

227 compilation_counter.num_compiled_artifacts_saved += 1

228 return compiled_graph, (key, path)

229

@@ -237,7 +240,7 @@ class InductorStandaloneAdaptor(CompilerInterface):

240 assert isinstance(handle[1], str)

241 path = handle[1]

242 inductor_compiled_graph = torch._inductor.CompiledArtifact.load(

240 - path=path, format="unpacked"

243 + path=path, format=self.save_format

244 )

245 from torch._inductor.compile_fx import graph_returns_tuple

246

vllm/config/compilation.py

+22 -1

@@ -7,11 +7,12 @@ from collections import Counter

7 from collections.abc import Callable

8 from dataclasses import asdict, field

9 from pathlib import Path

10 -from typing import TYPE_CHECKING, Any, ClassVar

10 +from typing import TYPE_CHECKING, Any, ClassVar, Literal

12 from pydantic import TypeAdapter, field_validator

13 from pydantic.dataclasses import dataclass

15 +import vllm.envs as envs

16 from vllm.compilation.inductor_pass import CallableInductorPass, InductorPass

17 from vllm.config.utils import config

18 from vllm.logger import init_logger

@@ -208,6 +209,15 @@ class CompilationConfig:

209 """The directory to store the compiled graph, to accelerate Inductor

210 compilation. By default, it will use model-related information to generate

211 a cache directory."""

212 + compile_cache_save_format: Literal["binary", "unpacked"] = field(

213 + default_factory=lambda: envs.VLLM_COMPILE_CACHE_SAVE_FORMAT

214 + )

215 + """Format for saving torch compile cache:\n

216 + - "binary": saves as binary file (multiprocess safe)\n

217 + - "unpacked": saves as directory structure for inspection/debugging

218 + (NOT multiprocess safe)\n

219 + Defaults to `VLLM_COMPILE_CACHE_SAVE_FORMAT` if not specified.

220 + """

221 backend: str = ""

222 """The backend for compilation. It needs to be a string:

223

@@ -479,6 +489,7 @@ class CompilationConfig:

489 factors.append(self.inductor_compile_config)

490 factors.append(self.inductor_passes)

491 factors.append(self.pass_config.uuid())

492 + factors.append(self.compile_cache_save_format)

493 return hashlib.sha256(str(factors).encode()).hexdigest()

494

495 def __repr__(self) -> str:

@@ -520,6 +531,16 @@ class CompilationConfig:

531 return CUDAGraphMode[value.upper()]

532 return value

533

534 + @field_validator("compile_cache_save_format")

535 + @classmethod

536 + def validate_compile_cache_save_format(cls, value: str) -> str:

537 + if value not in ("binary", "unpacked"):

538 + raise ValueError(

539 + f"compile_cache_save_format must be 'binary' or 'unpacked', "

540 + f"got: {value}"

541 + )

542 + return value

543 +

544 def __post_init__(self) -> None:

545 if self.level is not None:

546 logger.warning(

vllm/envs.py

+10

@@ -218,6 +218,7 @@ if TYPE_CHECKING:

218 VLLM_USE_FBGEMM: bool = False

219 VLLM_GC_DEBUG: str = ""

220 VLLM_DISABLE_SHARED_EXPERTS_STREAM: bool = False

221 + VLLM_COMPILE_CACHE_SAVE_FORMAT: Literal["binary", "unpacked"] = "binary"

222

223

224 def get_default_cache_root():

@@ -1442,6 +1443,15 @@ environment_variables: dict[str, Callable[[], Any]] = {

1443 "VLLM_DISABLE_SHARED_EXPERTS_STREAM": lambda: os.getenv(

1444 "VLLM_DISABLE_SHARED_EXPERTS_STREAM", False

1445 ),

1446 + # Format for saving torch.compile cache artifacts

1447 + # - "binary": saves as binary file

1448 + # Safe for multiple vllm serve processes accessing the same torch compile cache.

1449 + # - "unpacked": saves as directory structure (for inspection/debugging)

1450 + # NOT multiprocess safe - race conditions may occur with multiple processes.

1451 + # Allows viewing and setting breakpoints in Inductor's code output files.

1452 + "VLLM_COMPILE_CACHE_SAVE_FORMAT": env_with_choices(

1453 + "VLLM_COMPILE_CACHE_SAVE_FORMAT", "binary", ["binary", "unpacked"]

1454 + ),

1455 }

1456

1457 # --8<-- [end:env-vars-definition]