A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[torch.compile]: Add VLLM_DEBUG_DUMP_PATH environment variable (#25651)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Signed-off-by: Jiangyun Zhu <riverclouds.zhu@qq.com> Co-authored-by: Luka Govedič <ProExpertProg@users.noreply.github.com>
Jiangyun Zhu committed
Sep 28, 2025 at 00:09 UTC
c0ec81836fd47492a900a2538dea461619122555
6 files changed
+44
-17
vllm/compilation/monitor.py
+4
-6
@@ -1,7 +1,6 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
-import os
4
import time
5
6
from vllm.config import CompilationConfig, CompilationLevel, VllmConfig
@@ -18,13 +17,12 @@ def start_monitoring_torch_compile(vllm_config: VllmConfig):
17
torch_compile_start_time = time.time()
18
19
compilation_config: CompilationConfig = vllm_config.compilation_config
21
- if compilation_config.level == CompilationLevel.PIECEWISE and \
22
- compilation_config.debug_dump_path:
20
+ path = vllm_config.compile_debug_dump_path()
21
+ if compilation_config.level == CompilationLevel.PIECEWISE and path:
22
import depyf
24
- path = os.path.join(compilation_config.debug_dump_path,
25
- f"rank_{vllm_config.parallel_config.rank}")
23
+ path.mkdir(parents=True, exist_ok=True)
24
global context_manager
27
- context_manager = depyf.prepare_debug(path)
25
+ context_manager = depyf.prepare_debug(path.as_posix())
26
context_manager.__enter__()
27
28
vllm/compilation/vllm_inductor_pass.py
+1
-4
@@ -3,7 +3,6 @@
3
import functools
4
import operator
5
import time
6
-from pathlib import Path
6
from typing import ClassVar, Optional
7
8
import regex as re
@@ -96,12 +95,10 @@ class VllmPatternMatcherPass(VllmInductorPass):
95
96
TODO(luka): use pattern object to manually produce pattern graph
97
"""
99
- debug_dump_path = config.compilation_config.debug_dump_path
98
+ debug_dump_path = config.compile_debug_dump_path()
99
if not debug_dump_path:
100
return
101
103
- rank = config.parallel_config.rank
104
- debug_dump_path = Path(debug_dump_path) / f"rank_{rank}"
102
debug_dump_path.mkdir(parents=True, exist_ok=True)
103
104
from vllm.utils import unique_filepath
vllm/compilation/wrapper.py
+5
-6
@@ -92,12 +92,11 @@ class TorchCompileWrapperWithCustomDispatcher:
92
return
93
94
self.compiled_codes.append(new_code)
95
- debug_dump_dir = self.vllm_config.compilation_config.debug_dump_path
96
- if isinstance(debug_dump_dir, str) and debug_dump_dir != "":
97
- rank = self.vllm_config.parallel_config.rank
98
- decompiled_file = os.path.join(debug_dump_dir, f"rank_{rank}",
99
- "transformed_code.py")
100
- if not os.path.exists(decompiled_file):
95
+
96
+ path = self.vllm_config.compile_debug_dump_path()
97
+ if path:
98
+ decompiled_file = path / "transformed_code.py"
99
+ if not decompiled_file.exists():
100
try:
101
# usually the decompilation will succeed for most models,
102
# as we guarantee a full-graph compilation in Dynamo.
vllm/config/__init__.py
+26
@@ -12,6 +12,7 @@ import textwrap
12
from contextlib import contextmanager
13
from dataclasses import field, fields, is_dataclass, replace
14
from functools import cached_property, lru_cache
15
+from pathlib import Path
16
from typing import (TYPE_CHECKING, Any, Literal, Optional, Protocol, TypeVar,
17
Union, cast)
18
@@ -541,6 +542,17 @@ class VllmConfig:
542
# local attention.
543
self.scheduler_config.disable_hybrid_kv_cache_manager = True
544
545
+ if self.compilation_config.debug_dump_path:
546
+ self.compilation_config.debug_dump_path = \
547
+ self.compilation_config.debug_dump_path.absolute().expanduser()
548
+ if envs.VLLM_DEBUG_DUMP_PATH is not None:
549
+ env_path = Path(envs.VLLM_DEBUG_DUMP_PATH).absolute().expanduser()
550
+ if self.compilation_config.debug_dump_path:
551
+ logger.warning(
552
+ "Config-specified debug dump path is overridden"
553
+ " by VLLM_DEBUG_DUMP_PATH to %s", env_path)
554
+ self.compilation_config.debug_dump_path = env_path
555
+
556
def update_sizes_for_sequence_parallelism(self,
557
possible_sizes: list) -> list:
558
# remove the sizes that not multiple of tp_size when
@@ -672,6 +684,20 @@ class VllmConfig:
684
f"but got '{self.load_config.load_format}'. "
685
f"Model: {self.model_config.model}")
686
687
+ def compile_debug_dump_path(self) -> Optional[Path]:
688
+ """Returns a rank-aware path for dumping
689
+ torch.compile debug information.
690
+ """
691
+ if self.compilation_config.debug_dump_path is None:
692
+ return None
693
+ tp_rank = self.parallel_config.rank
694
+ dp_rank = self.parallel_config.data_parallel_rank
695
+ data_parallel_size = self.parallel_config.data_parallel_size
696
+ append_path = f"rank_{tp_rank}" if data_parallel_size == 1 \
697
+ else f"rank_{tp_rank}_dp_{dp_rank}"
698
+ path = self.compilation_config.debug_dump_path / append_path
699
+ return path
700
+
701
def __str__(self):
702
return (
703
f"model={self.model_config.model!r}, "
vllm/config/compilation.py
+2
-1
@@ -5,6 +5,7 @@ import enum
5
import hashlib
6
from collections import Counter
7
from dataclasses import asdict, field
8
+from pathlib import Path
9
from typing import TYPE_CHECKING, Any, Callable, ClassVar, Optional, Union
10
11
from pydantic import TypeAdapter, field_validator
@@ -169,7 +170,7 @@ class CompilationConfig:
170
- 1: dynamo as is.
171
- 2: dynamo once.
172
- 3: piecewise compilation."""
172
- debug_dump_path: str = ""
173
+ debug_dump_path: Optional[Path] = None
174
"""The path to dump the debug information."""
175
cache_dir: str = ""
176
"""The directory to store the compiled graph, to accelerate Inductor
vllm/envs.py
+6
@@ -199,6 +199,7 @@ if TYPE_CHECKING:
199
VLLM_DBO_COMM_SMS: int = 20
200
GPT_OSS_SYSTEM_TOOL_MCP_LABELS: list[str] = []
201
VLLM_PATTERN_MATCH_DEBUG: Optional[str] = None
202
+ VLLM_DEBUG_DUMP_PATH: Optional[str] = None
203
VLLM_ENABLE_INDUCTOR_MAX_AUTOTUNE: bool = True
204
VLLM_ENABLE_INDUCTOR_COORDINATE_DESCENT_TUNING: bool = True
205
VLLM_USE_NCCL_SYMM_MEM: bool = False
@@ -513,6 +514,11 @@ environment_variables: dict[str, Callable[[], Any]] = {
514
"VLLM_PATTERN_MATCH_DEBUG":
515
lambda: os.environ.get("VLLM_PATTERN_MATCH_DEBUG", None),
516
517
+ # Dump fx graphs to the given directory.
518
+ # It will override CompilationConfig.debug_dump_path if set.
519
+ "VLLM_DEBUG_DUMP_PATH":
520
+ lambda: os.environ.get("VLLM_DEBUG_DUMP_PATH", None),
521
+
522
# local rank of the process in the distributed setting, used to determine
523
# the GPU device id
524
"LOCAL_RANK":