@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 88a412ed3d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[torch.compile] fast inductor (#11108)

Signed-off-by: youkaichao <youkaichao@gmail.com> Co-authored-by: Tyler Michael Smith <tyler@neuralmagic.com>

youkaichao committed Dec 16, 2024 at 16:15 UTC 88a412ed3d964de3443c42a6a35108115ee0ad25
3 files changed +624 -7
vllm/compilation/backends.py
+210 -3
@@ -1,6 +1,10 @@
1 +import ast
2 import copy
3 import dataclasses
4 +import os
5 +import pprint
6 import time
7 +from collections import defaultdict
8 from contextlib import ExitStack
9 from typing import Any, Callable, Dict, List, Optional, Sequence, Set, Tuple
10 from unittest.mock import patch
@@ -21,6 +25,122 @@ from .pass_manager import PostGradPassManager
25 logger = init_logger(__name__)
26
27
28 +class InductorHashCache:
29 + """
30 + Disk format: a Python list of tuples, each tuple is
31 + (runtime_shape, graph_index, hash_str)
32 + We use list of tuple for readability.
33 +
34 + In-memory format: a defaultdict of dict, where the key is
35 + runtime_shape, and the value is a dict of graph_index to hash_str.
36 +
37 + The data is essentially `Dict[Optional[int], Dict[int, str]]`,
38 + we don't use json here because json doesn't support int as key.
39 +
40 + TODO: better off-the-shelf solution to serialize the data?
41 + """
42 +
43 + def __init__(self, cache_dir: str, disabled: bool = False):
44 + self.cache: defaultdict = defaultdict(dict)
45 + self.disabled = disabled
46 + self.cache_dir = cache_dir
47 + self.cache_file_path = os.path.join(cache_dir,
48 + "inductor_hash_cache.py")
49 + if disabled:
50 + return
51 + # set flags so that Inductor and Triton store their cache
52 + # in the cache_dir, then users only need to copy the cache_dir
53 + # to another machine to reuse the cache.
54 + inductor_cache = os.path.join(cache_dir, "inductor_cache")
55 + os.makedirs(inductor_cache, exist_ok=True)
56 + os.environ["TORCHINDUCTOR_CACHE_DIR"] = inductor_cache
57 + triton_cache = os.path.join(cache_dir, "triton_cache")
58 + os.makedirs(triton_cache, exist_ok=True)
59 + os.environ["TRITON_CACHE_DIR"] = triton_cache
60 + if os.path.exists(self.cache_file_path):
61 + with open(self.cache_file_path) as f:
62 + self.deserialize(f.read())
63 +
64 + def deserialize(self, data: str):
65 + # we use ast.literal_eval to parse the data
66 + # because it is a safe way to parse Python literals.
67 + # do not use eval(), it is unsafe.
68 + list_data = ast.literal_eval(data)
69 + for runtime_shape, graph_index, hash_str in list_data:
70 + self.cache[runtime_shape][graph_index] = hash_str
71 +
72 + def serialize(self) -> str:
73 + data = []
74 + for runtime_shape, graph_index_to_hash_str in self.cache.items():
75 + for graph_index, hash_str in graph_index_to_hash_str.items():
76 + data.append((runtime_shape, graph_index, hash_str))
77 + printer = pprint.PrettyPrinter(indent=4)
78 + return printer.pformat(data)
79 +
80 + def save_to_file(self):
81 + if self.disabled:
82 + return
83 + with open(self.cache_file_path, "w") as f:
84 + f.write(self.serialize())
85 +
86 + def __contains__(self, key: Tuple[Optional[int], int]) -> bool:
87 + if self.disabled:
88 + return False
89 + runtime_shape, graph_index = key
90 + return runtime_shape in self.cache and graph_index in self.cache[
91 + runtime_shape]
92 +
93 + def __getitem__(self, key: Tuple[Optional[int], int]) -> str:
94 + if self.disabled:
95 + raise KeyError("cannot read from disabled cache")
96 + runtime_shape, graph_index = key
97 + return self.cache[runtime_shape][graph_index]
98 +
99 + def __setitem__(self, key: Tuple[Optional[int], int], value: str):
100 + # setitem for disabled cache is fine, because we
101 + # don't actually write to the disk
102 + runtime_shape, graph_index = key
103 + self.cache[runtime_shape][graph_index] = value
104 +
105 +
106 +class AlwaysHitShapeEnv:
107 + """
108 + Why do we need this class:
109 +
110 + For normal `torch.compile` usage, every compilation will have
111 + one Dynamo bytecode compilation and one Inductor compilation.
112 + The Inductor compilation happens under the context of the
113 + Dynamo bytecode compilation, and that context is used to
114 + determine the dynamic shape information, etc.
115 +
116 + For our use case, we only run Dynamo bytecode compilation once,
117 + and run Inductor compilation multiple times with different shapes
118 + plus a general shape. The compilation for specific shapes happens
119 + outside of the context of the Dynamo bytecode compilation. At that
120 + time, we don't have shape environment to provide to Inductor, and
121 + it will fail the Inductor code cache lookup.
122 +
123 + By providing a dummy shape environment that always hits, we can
124 + make the Inductor code cache lookup always hit, and we can
125 + compile the graph for different shapes as needed.
126 +
127 + The following dummy methods are obtained by trial-and-error
128 + until it works.
129 + """
130 +
131 + def __init__(self) -> None:
132 + self.guards: List[Any] = []
133 +
134 + def evaluate_guards_expression(self, *args, **kwargs):
135 + return True
136 +
137 + def get_pruned_guards(self, *args, **kwargs):
138 + return []
139 +
140 + def produce_guards_expression(self, *args, **kwargs):
141 + return ""
142 +
143 +
144 def wrap_inductor(graph,
145 example_inputs,
146 additional_inductor_config,
@@ -55,9 +175,93 @@ def wrap_inductor(graph,
175 # inductor can inplace modify the graph, so we need to copy it
176 # see https://github.com/pytorch/pytorch/issues/138980
177 graph = copy.deepcopy(graph)
58 - compiled_graph = compile_fx(graph,
59 - example_inputs,
60 - config_patches=current_config)
178 +
179 + cache_data = compilation_config.inductor_hash_cache
180 + if (runtime_shape, graph_index) in cache_data:
181 + # we compiled this graph before
182 + # so we can directly lookup the compiled graph via hash
183 + hash_str = cache_data[(runtime_shape, graph_index)]
184 + if graph_index == 0:
185 + # adds some info logging for the first graph
186 + logger.info(
187 + "Directly lookup the graph for shape %s from the cache",
188 + str(runtime_shape)) # noqa
189 + logger.debug(
190 + "directly lookup the %s-th graph for shape %s via hash %s",
191 + graph_index, str(runtime_shape), hash_str)
192 + from torch._inductor.codecache import FxGraphCache
193 + with patch("torch._inductor.codecache.FxGraphCache._get_shape_env",
194 + lambda *args, **kwargs: AlwaysHitShapeEnv()):
195 + inductor_compiled_graph = FxGraphCache._lookup_graph(
196 + hash_str, example_inputs, True, False)
197 + assert inductor_compiled_graph is not None, (
198 + "Inductor cache lookup failed. Please remove"
199 + f"the cache file {compilation_config.inductor_hash_cache.cache_file_path} and try again." # noqa
200 + )
201 +
202 + # Inductor calling convention (function signature):
203 + # f(list) -> tuple
204 + # Dynamo calling convention (function signature):
205 + # f(*args) -> Any
206 +
207 + # need to know if the graph returns a tuple
208 + from torch._inductor.compile_fx import graph_returns_tuple
209 + returns_tuple = graph_returns_tuple(graph)
210 +
211 + # this is the graph we return to Dynamo to run
212 + def compiled_graph(*args):
213 + # convert args to list
214 + list_args = list(args)
215 + graph_output = inductor_compiled_graph(list_args)
216 + # unpack the tuple if needed
217 + if returns_tuple:
218 + return graph_output
219 + else:
220 + return graph_output[0]
221 + else:
222 + # it's the first time we compile this graph
223 + # the assumption is that we don't have nested Inductor compilation.
224 + # compiled_fx_graph_hash will only be called once, and we can hook
225 + # it to get the hash of the compiled graph directly.
226 + from torch._inductor.codecache import compiled_fx_graph_hash
227 +
228 + def hijack_compiled_fx_graph_hash(*args, **kwargs):
229 + out = compiled_fx_graph_hash(*args, **kwargs)
230 + # store the hash in the cache
231 + nonlocal cache_data
232 + cache_data[(runtime_shape, graph_index)] = out[0]
233 + if graph_index == 0:
234 + # adds some info logging for the first graph
235 + logger.info("Cache the graph of shape %s for later use",
236 + str(runtime_shape))
237 + logger.debug("store the %s-th graph for shape %s via hash %s",
238 + graph_index, str(runtime_shape), out[0])
239 + return out
240 +
241 + def _check_can_cache(*args, **kwargs):
242 + # no error means it can be cached.
243 + # Inductor refuses to cache the graph outside of Dynamo
244 + # tracing context, and also disables caching for graphs
245 + # with high-order ops.
246 + # For vLLM, in either case, we want to cache the graph.
247 + # see https://github.com/pytorch/pytorch/blob/9f5ebf3fc609105a74eab4ccc24932d6353ff566/torch/_inductor/codecache.py#L1221 # noqa
248 + return
249 +
250 + def _get_shape_env():
251 + return AlwaysHitShapeEnv()
252 +
253 + with patch(# for hijacking the hash of the compiled graph
254 + "torch._inductor.codecache.compiled_fx_graph_hash",
255 + hijack_compiled_fx_graph_hash), \
256 + patch(# for providing a dummy shape environment
257 + "torch._inductor.codecache.FxGraphCache._get_shape_env",
258 + _get_shape_env), \
259 + patch(# for forcing the graph to be cached
260 + "torch._inductor.codecache.FxGraphCache._check_can_cache",
261 + _check_can_cache):
262 + compiled_graph = compile_fx(graph,
263 + example_inputs,
264 + config_patches=current_config)
265
266 # after compiling the last graph, record the end time
267 if graph_index == num_graphs - 1:
@@ -457,6 +661,9 @@ class PiecewiseBackend:
661
662 # finished compilations for all required shapes
663 if self.is_last_graph and not self.to_be_compiled_sizes:
664 +
665 + # save the hash of the inductor graph for the next run
666 + self.compilation_config.inductor_hash_cache.save_to_file()
667 end_monitoring_torch_compile(self.vllm_config)
668
669 if not entry.use_cudagraph:
vllm/config.py
+411 -4
@@ -3,6 +3,7 @@ import copy
3 import enum
4 import hashlib
5 import json
6 +import os
7 import warnings
8 from contextlib import contextmanager
9 from dataclasses import dataclass, field, replace
@@ -162,6 +163,30 @@ class ModelConfig:
163 which allows no processors.
164 """
165
166 + def compute_hash(self) -> str:
167 + """
168 + WARNING: Whenever a new field is added to this config,
169 + ensure that it is included in the factors list if
170 + it affects the computation graph.
171 +
172 + Provide a hash that uniquely identifies all the configs
173 + that affect the structure of the computation
174 + graph from input ids/embeddings to the final hidden states,
175 + excluding anything before input ids/embeddings and after
176 + the final hidden states.
177 + """
178 + factors: List[Any] = []
179 + factors.append(self.model)
180 + factors.append(self.dtype)
181 + factors.append(self.quantization)
182 + factors.append(self.quantization_param_path)
183 + factors.append(self.revision)
184 + factors.append(self.code_revision)
185 + factors.append(self.trust_remote_code)
186 + factors.append(self.rope_scaling)
187 + factors.append(self.rope_theta)
188 + return hashlib.sha256(str(factors).encode()).hexdigest()
189 +
190 def __init__(self,
191 model: str,
192 task: Union[TaskOption, Literal["draft"]],
@@ -203,6 +228,8 @@ class ModelConfig:
228 self.seed = seed
229 self.revision = revision
230 self.code_revision = code_revision
231 + self.rope_scaling = rope_scaling
232 + self.rope_theta = rope_theta
233
234 if hf_overrides is None:
235 hf_overrides = {}
@@ -832,6 +859,24 @@ class CacheConfig:
859 cpu_offload_gb: Size of the CPU offload buffer in GiB.
860 """
861
862 + def compute_hash(self) -> str:
863 + """
864 + WARNING: Whenever a new field is added to this config,
865 + ensure that it is included in the factors list if
866 + it affects the computation graph.
867 +
868 + Provide a hash that uniquely identifies all the configs
869 + that affect the structure of the computation
870 + graph from input ids/embeddings to the final hidden states,
871 + excluding anything before input ids/embeddings and after
872 + the final hidden states.
873 + """
874 + factors: List[Any] = []
875 + factors.append(self.cache_dtype)
876 + # `cpu_offload_gb` does not use `torch.compile` yet.
877 + hash_str = hashlib.md5(str(factors).encode()).hexdigest()
878 + return hash_str
879 +
880 def __init__(
881 self,
882 block_size: int,
@@ -928,6 +973,24 @@ class TokenizerPoolConfig:
973 pool_type: Union[str, Type["BaseTokenizerGroup"]]
974 extra_config: dict
975
976 + def compute_hash(self) -> str:
977 + """
978 + WARNING: Whenever a new field is added to this config,
979 + ensure that it is included in the factors list if
980 + it affects the computation graph.
981 +
982 + Provide a hash that uniquely identifies all the configs
983 + that affect the structure of the computation
984 + graph from input ids/embeddings to the final hidden states,
985 + excluding anything before input ids/embeddings and after
986 + the final hidden states.
987 + """
988 + # no factors to consider.
989 + # this config will not affect the computation graph.
990 + factors: List[Any] = []
991 + hash_str = hashlib.md5(str(factors).encode()).hexdigest()
992 + return hash_str
993 +
994 def __post_init__(self):
995 if self.pool_type not in ("ray", ) and not isinstance(
996 self.pool_type, type):
@@ -1010,6 +1073,24 @@ class LoadConfig:
1073 default_factory=dict)
1074 ignore_patterns: Optional[Union[List[str], str]] = None
1075
1076 + def compute_hash(self) -> str:
1077 + """
1078 + WARNING: Whenever a new field is added to this config,
1079 + ensure that it is included in the factors list if
1080 + it affects the computation graph.
1081 +
1082 + Provide a hash that uniquely identifies all the configs
1083 + that affect the structure of the computation
1084 + graph from input ids/embeddings to the final hidden states,
1085 + excluding anything before input ids/embeddings and after
1086 + the final hidden states.
1087 + """
1088 + # no factors to consider.
1089 + # this config will not affect the computation graph.
1090 + factors: List[Any] = []
1091 + hash_str = hashlib.md5(str(factors).encode()).hexdigest()
1092 + return hash_str
1093 +
1094 def __post_init__(self):
1095 model_loader_extra_config = self.model_loader_extra_config or {}
1096 if isinstance(model_loader_extra_config, str):
@@ -1073,6 +1154,19 @@ class ParallelConfig:
1154
1155 rank: int = 0
1156
1157 + def compute_hash(self):
1158 + """
1159 + Provide a hash that uniquely identifies all the configs
1160 + that affect the structure of the computation
1161 + graph from input ids/embeddings to the final hidden states,
1162 + excluding anything before input ids/embeddings and after
1163 + the final hidden states.
1164 + """
1165 + factors: List[Any] = []
1166 + factors.append(self.pipeline_parallel_size)
1167 + factors.append(self.tensor_parallel_size)
1168 + return hashlib.sha256(str(factors).encode()).hexdigest()
1169 +
1170 def __post_init__(self) -> None:
1171 self.world_size = self.pipeline_parallel_size * \
1172 self.tensor_parallel_size
@@ -1209,6 +1303,24 @@ class SchedulerConfig:
1303
1304 chunked_prefill_enabled: bool = field(init=False)
1305
1306 + def compute_hash(self) -> str:
1307 + """
1308 + WARNING: Whenever a new field is added to this config,
1309 + ensure that it is included in the factors list if
1310 + it affects the computation graph.
1311 +
1312 + Provide a hash that uniquely identifies all the configs
1313 + that affect the structure of the computation
1314 + graph from input ids/embeddings to the final hidden states,
1315 + excluding anything before input ids/embeddings and after
1316 + the final hidden states.
1317 + """
1318 + # no factors to consider.
1319 + # this config will not affect the computation graph.
1320 + factors: List[Any] = []
1321 + hash_str = hashlib.md5(str(factors).encode()).hexdigest()
1322 + return hash_str
1323 +
1324 def __post_init__(self) -> None:
1325 if self.max_num_batched_tokens is None:
1326 if self.enable_chunked_prefill:
@@ -1286,6 +1398,25 @@ class DeviceConfig:
1398 device: Optional[torch.device]
1399 device_type: str
1400
1401 + def compute_hash(self) -> str:
1402 + """
1403 + WARNING: Whenever a new field is added to this config,
1404 + ensure that it is included in the factors list if
1405 + it affects the computation graph.
1406 +
1407 + Provide a hash that uniquely identifies all the configs
1408 + that affect the structure of the computation
1409 + graph from input ids/embeddings to the final hidden states,
1410 + excluding anything before input ids/embeddings and after
1411 + the final hidden states.
1412 + """
1413 + # no factors to consider.
1414 + # the device/platform information will be summarized
1415 + # by torch/vllm automatically.
1416 + factors: List[Any] = []
1417 + hash_str = hashlib.md5(str(factors).encode()).hexdigest()
1418 + return hash_str
1419 +
1420 def __init__(self, device: str = "auto") -> None:
1421 if device == "auto":
1422 # Automated device type detection
@@ -1313,6 +1444,24 @@ class SpeculativeConfig:
1444 decoding with top-1 proposals.
1445 """
1446
1447 + def compute_hash(self) -> str:
1448 + """
1449 + WARNING: Whenever a new field is added to this config,
1450 + ensure that it is included in the factors list if
1451 + it affects the computation graph.
1452 +
1453 + Provide a hash that uniquely identifies all the configs
1454 + that affect the structure of the computation
1455 + graph from input ids/embeddings to the final hidden states,
1456 + excluding anything before input ids/embeddings and after
1457 + the final hidden states.
1458 + """
1459 + # no factors to consider.
1460 + # spec decode does not use `torch.compile` yet.
1461 + factors: List[Any] = []
1462 + hash_str = hashlib.md5(str(factors).encode()).hexdigest()
1463 + return hash_str
1464 +
1465 @staticmethod
1466 def maybe_create_spec_config(
1467 target_model_config: ModelConfig,
@@ -1753,6 +1902,24 @@ class LoRAConfig:
1902 long_lora_scaling_factors: Optional[Tuple[float]] = None
1903 bias_enabled: bool = False
1904
1905 + def compute_hash(self) -> str:
1906 + """
1907 + WARNING: Whenever a new field is added to this config,
1908 + ensure that it is included in the factors list if
1909 + it affects the computation graph.
1910 +
1911 + Provide a hash that uniquely identifies all the configs
1912 + that affect the structure of the computation
1913 + graph from input ids/embeddings to the final hidden states,
1914 + excluding anything before input ids/embeddings and after
1915 + the final hidden states.
1916 + """
1917 + # no factors to consider.
1918 + # LoRA is not compatible with `torch.compile` .
1919 + factors: List[Any] = []
1920 + hash_str = hashlib.md5(str(factors).encode()).hexdigest()
1921 + return hash_str
1922 +
1923 def __post_init__(self):
1924 # Setting the maximum rank to 256 should be able to satisfy the vast
1925 # majority of applications.
@@ -1802,6 +1969,24 @@ class PromptAdapterConfig:
1969 max_cpu_prompt_adapters: Optional[int] = None
1970 prompt_adapter_dtype: Optional[torch.dtype] = None
1971
1972 + def compute_hash(self) -> str:
1973 + """
1974 + WARNING: Whenever a new field is added to this config,
1975 + ensure that it is included in the factors list if
1976 + it affects the computation graph.
1977 +
1978 + Provide a hash that uniquely identifies all the configs
1979 + that affect the structure of the computation
1980 + graph from input ids/embeddings to the final hidden states,
1981 + excluding anything before input ids/embeddings and after
1982 + the final hidden states.
1983 + """
1984 + # no factors to consider.
1985 + # this config will not affect the computation graph.
1986 + factors: List[Any] = []
1987 + hash_str = hashlib.md5(str(factors).encode()).hexdigest()
1988 + return hash_str
1989 +
1990 def __post_init__(self):
1991
1992 if self.max_prompt_adapters < 1:
@@ -1830,6 +2015,24 @@ class MultiModalConfig:
2015 for each :class:`~vllm.multimodal.MultiModalPlugin`.
2016 """
2017
2018 + def compute_hash(self) -> str:
2019 + """
2020 + WARNING: Whenever a new field is added to this config,
2021 + ensure that it is included in the factors list if
2022 + it affects the computation graph.
2023 +
2024 + Provide a hash that uniquely identifies all the configs
2025 + that affect the structure of the computation
2026 + graph from input ids/embeddings to the final hidden states,
2027 + excluding anything before input ids/embeddings and after
2028 + the final hidden states.
2029 + """
2030 + # no factors to consider.
2031 + # this config will not affect the computation graph.
2032 + factors: List[Any] = []
2033 + hash_str = hashlib.md5(str(factors).encode()).hexdigest()
2034 + return hash_str
2035 +
2036 # TODO: Add configs to init vision tower or not.
2037
2038
@@ -1869,6 +2072,24 @@ class PoolerConfig:
2072 ``math-shepherd-mistral-7b-prm`` model.
2073 """
2074
2075 + def compute_hash(self) -> str:
2076 + """
2077 + WARNING: Whenever a new field is added to this config,
2078 + ensure that it is included in the factors list if
2079 + it affects the computation graph.
2080 +
2081 + Provide a hash that uniquely identifies all the configs
2082 + that affect the structure of the computation
2083 + graph from input ids/embeddings to the final hidden states,
2084 + excluding anything before input ids/embeddings and after
2085 + the final hidden states.
2086 + """
2087 + # no factors to consider.
2088 + # this config will not affect the computation graph.
2089 + factors: List[Any] = []
2090 + hash_str = hashlib.md5(str(factors).encode()).hexdigest()
2091 + return hash_str
2092 +
2093 @staticmethod
2094 def from_json(json_str: str) -> "PoolerConfig":
2095 return PoolerConfig(**json.loads(json_str))
@@ -2103,6 +2324,24 @@ class DecodingConfig:
2324 # 'outlines' / 'lm-format-enforcer' / 'xgrammar'
2325 guided_decoding_backend: str = 'xgrammar'
2326
2327 + def compute_hash(self) -> str:
2328 + """
2329 + WARNING: Whenever a new field is added to this config,
2330 + ensure that it is included in the factors list if
2331 + it affects the computation graph.
2332 +
2333 + Provide a hash that uniquely identifies all the configs
2334 + that affect the structure of the computation
2335 + graph from input ids/embeddings to the final hidden states,
2336 + excluding anything before input ids/embeddings and after
2337 + the final hidden states.
2338 + """
2339 + # no factors to consider.
2340 + # this config will not affect the computation graph.
2341 + factors: List[Any] = []
2342 + hash_str = hashlib.md5(str(factors).encode()).hexdigest()
2343 + return hash_str
2344 +
2345 def __post_init__(self):
2346 valid_guided_backends = ['outlines', 'lm-format-enforcer', 'xgrammar']
2347 backend = self.guided_decoding_backend
@@ -2124,6 +2363,24 @@ class ObservabilityConfig:
2363 # If set, collects the model execute time for the request.
2364 collect_model_execute_time: bool = False
2365
2366 + def compute_hash(self) -> str:
2367 + """
2368 + WARNING: Whenever a new field is added to this config,
2369 + ensure that it is included in the factors list if
2370 + it affects the computation graph.
2371 +
2372 + Provide a hash that uniquely identifies all the configs
2373 + that affect the structure of the computation
2374 + graph from input ids/embeddings to the final hidden states,
2375 + excluding anything before input ids/embeddings and after
2376 + the final hidden states.
2377 + """
2378 + # no factors to consider.
2379 + # this config will not affect the computation graph.
2380 + factors: List[Any] = []
2381 + hash_str = hashlib.md5(str(factors).encode()).hexdigest()
2382 + return hash_str
2383 +
2384 def __post_init__(self):
2385 if not is_otel_available() and self.otlp_traces_endpoint is not None:
2386 raise ValueError(
@@ -2165,6 +2422,24 @@ class KVTransferConfig(BaseModel):
2422 # The KV connector port, used to build distributed connection
2423 kv_port: int = 14579
2424
2425 + def compute_hash(self) -> str:
2426 + """
2427 + WARNING: Whenever a new field is added to this config,
2428 + ensure that it is included in the factors list if
2429 + it affects the computation graph.
2430 +
2431 + Provide a hash that uniquely identifies all the configs
2432 + that affect the structure of the computation
2433 + graph from input ids/embeddings to the final hidden states,
2434 + excluding anything before input ids/embeddings and after
2435 + the final hidden states.
2436 + """
2437 + # no factors to consider.
2438 + # this config will not affect the computation graph.
2439 + factors: List[Any] = []
2440 + hash_str = hashlib.md5(str(factors).encode()).hexdigest()
2441 + return hash_str
2442 +
2443 @classmethod
2444 def from_cli(cls, cli_value: str) -> "KVTransferConfig":
2445 """Parse the CLI value for the kv cache transfer config."""
@@ -2234,6 +2509,9 @@ class CompilationConfig(BaseModel):
2509 - 2: dynamo once.
2510 - 3: piecewise compilation.
2511 - debug_dump_path: the path to dump the debug information.
2512 + - cache_dir: the directory to store the compiled graph, to
2513 + accelerate Inductor compilation. By default, it will use
2514 + model-related information to generate a cache directory.
2515 - backend: the backend for compilation. It needs to be a string.
2516 - "" (empty string): use the default backend.
2517 - "eager"/"openxla"/...: use the specified backend registered in PyTorch.
@@ -2302,12 +2580,10 @@ class CompilationConfig(BaseModel):
2580 """ # noqa
2581 level: int = 0
2582 debug_dump_path: str = ""
2583 + cache_dir: str = ""
2584 backend: str = ""
2585 custom_ops: List[str] = Field(default_factory=list)
2307 - splitting_ops: List[str] = Field(default_factory=lambda: [
2308 - "vllm.unified_attention",
2309 - "vllm.unified_attention_with_output",
2310 - ])
2586 + splitting_ops: List[str] = Field(default=None) # type: ignore
2587
2588 use_inductor: bool = True
2589 candidate_compile_sizes: Optional[List[int]] = Field(default=None)
@@ -2371,12 +2647,37 @@ class CompilationConfig(BaseModel):
2647 enabled_custom_ops: Counter[str] = PrivateAttr
2648 disabled_custom_ops: Counter[str] = PrivateAttr
2649 compilation_time: float = PrivateAttr
2650 + # should be InductorHashCache, but Pydantic does not support it
2651 + inductor_hash_cache: Any = PrivateAttr
2652
2653 # Per-model forward context
2654 # Mainly used to store attention cls
2655 # Map from layer name to the attention cls
2656 static_forward_context: Dict[str, Any] = PrivateAttr
2657
2658 + def compute_hash(self) -> str:
2659 + """
2660 + WARNING: Whenever a new field is added to this config,
2661 + ensure that it is included in the factors list if
2662 + it affects the computation graph.
2663 +
2664 + Provide a hash that uniquely identifies all the configs
2665 + that affect the structure of the computation
2666 + graph from input ids/embeddings to the final hidden states,
2667 + excluding anything before input ids/embeddings and after
2668 + the final hidden states.
2669 + """
2670 + factors: List[Any] = []
2671 + factors.append(self.level)
2672 + factors.append(self.backend)
2673 + factors.append(self.custom_ops)
2674 + factors.append(self.splitting_ops)
2675 + factors.append(self.use_inductor)
2676 + factors.append(self.inductor_compile_config)
2677 + factors.append(self.inductor_passes)
2678 + factors.append(self.pass_config.uuid())
2679 + return hashlib.sha256(str(factors).encode()).hexdigest()
2680 +
2681 def __repr__(self) -> str:
2682 exclude = {
2683 "static_forward_context",
@@ -2405,6 +2706,27 @@ class CompilationConfig(BaseModel):
2706 count_all = self.custom_ops.count("all")
2707 assert count_none + count_all <= 1, "Can only specify 'none' or 'all'"
2708
2709 + if self.splitting_ops is None:
2710 + if envs.VLLM_USE_V1:
2711 + # v1 must split the graph on attention ops
2712 + # for piecewise cudagraph
2713 + self.splitting_ops = [
2714 + "vllm.unified_attention",
2715 + "vllm.unified_attention_with_output",
2716 + ]
2717 + else:
2718 + # v0 can use full graph compilation without splitting,
2719 + # splitting is optional.
2720 + # right now we still need it. kv cache shape
2721 + # will be included in the graph if we don't split
2722 + # the graph.
2723 + # TODO: hide kv cache in static forward context
2724 + # so that inductor does not see it.
2725 + self.splitting_ops = [
2726 + "vllm.unified_attention",
2727 + "vllm.unified_attention_with_output",
2728 + ]
2729 +
2730 for k, v in self.inductor_passes.items():
2731 if not isinstance(v, str):
2732 assert callable(v), (
@@ -2444,6 +2766,30 @@ class CompilationConfig(BaseModel):
2766 # TODO: pass user-specified backend to piecewise compilation
2767 # merge with the config use_inductor
2768 assert self.level == CompilationLevel.PIECEWISE
2769 +
2770 + if not self.cache_dir:
2771 + # no provided cache dir, generate one based on the known factors
2772 + # that affects the compilation. if none of the factors change,
2773 + # the cache dir will be the same so that we can reuse the compiled
2774 + # graph.
2775 + hash_key = vllm_config.compute_hash()
2776 + cache_dir = os.path.join(
2777 + envs.VLLM_CACHE_ROOT, "torch_compile_cache", hash_key,
2778 + f"rank_{vllm_config.parallel_config.rank}")
2779 + os.makedirs(cache_dir, exist_ok=True)
2780 + self.cache_dir = cache_dir
2781 +
2782 + disabled = envs.VLLM_DISABLE_COMPILE_CACHE
2783 + from vllm.compilation.backends import InductorHashCache
2784 + self.inductor_hash_cache: InductorHashCache = InductorHashCache(
2785 + self.cache_dir, disabled=disabled)
2786 + if disabled:
2787 + logger.info("vLLM's torch.compile cache is disabled.")
2788 + else:
2789 + logger.info(
2790 + "Using cache directory: %s for vLLM's torch.compile",
2791 + self.cache_dir)
2792 +
2793 from vllm.compilation.backends import VllmBackend
2794 return VllmBackend(vllm_config)
2795
@@ -2520,6 +2866,67 @@ class VllmConfig:
2866 init=True) # type: ignore
2867 instance_id: str = ""
2868
2869 + def compute_hash(self) -> str:
2870 + """
2871 + WARNING: Whenever a new field is added to this config,
2872 + ensure that it is included in the factors list if
2873 + it affects the computation graph.
2874 +
2875 + Provide a hash that uniquely identifies all the configs
2876 + that affect the structure of the computation
2877 + graph from input ids/embeddings to the final hidden states,
2878 + excluding anything before input ids/embeddings and after
2879 + the final hidden states.
2880 + """
2881 + factors: List[Any] = []
2882 + # summarize system state
2883 + from torch._inductor.codecache import CacheBase
2884 + system_factors = CacheBase.get_system()
2885 + factors.append(system_factors)
2886 +
2887 + # summarize pytorch state
2888 + from torch._inductor.codecache import torch_key
2889 + torch_factors = torch_key()
2890 + factors.append(torch_factors)
2891 +
2892 + # summarize vllm config
2893 + vllm_factors: List[Any] = []
2894 + from vllm import __version__
2895 + vllm_factors.append(__version__)
2896 + if self.model_config:
2897 + vllm_factors.append(self.model_config.compute_hash())
2898 + if self.cache_config:
2899 + vllm_factors.append(self.cache_config.compute_hash())
2900 + if self.parallel_config:
2901 + vllm_factors.append(self.parallel_config.compute_hash())
2902 + if self.scheduler_config:
2903 + vllm_factors.append(self.scheduler_config.compute_hash())
2904 + if self.device_config:
2905 + vllm_factors.append(self.device_config.compute_hash())
2906 + if self.load_config:
2907 + vllm_factors.append(self.load_config.compute_hash())
2908 + if self.lora_config:
2909 + vllm_factors.append(self.lora_config.compute_hash())
2910 + if self.speculative_config:
2911 + vllm_factors.append(self.speculative_config.compute_hash())
2912 + if self.decoding_config:
2913 + vllm_factors.append(self.decoding_config.compute_hash())
2914 + if self.observability_config:
2915 + vllm_factors.append(self.observability_config.compute_hash())
2916 + if self.prompt_adapter_config:
2917 + vllm_factors.append(self.prompt_adapter_config.compute_hash())
2918 + if self.quant_config:
2919 + pass # should be captured by model_config.quantization
2920 + if self.compilation_config:
2921 + vllm_factors.append(self.compilation_config.compute_hash())
2922 + if self.kv_transfer_config:
2923 + vllm_factors.append(self.kv_transfer_config.compute_hash())
2924 +
2925 + factors.append(vllm_factors)
2926 +
2927 + hash_str = hashlib.md5(str(factors).encode()).hexdigest()[:10]
2928 + return hash_str
2929 +
2930 def pad_for_cudagraph(self, batch_size: int) -> int:
2931 # if batch_size > self.compilation_config.max_capture_size,
2932 # it should raise an IndexError.
vllm/envs.py
+3
@@ -71,6 +71,7 @@ if TYPE_CHECKING:
71 VLLM_USE_V1: bool = False
72 VLLM_ENABLE_V1_MULTIPROCESSING: bool = True
73 VLLM_LOG_BATCHSIZE_INTERVAL: float = -1
74 + VLLM_DISABLE_COMPILE_CACHE: bool = False
75
76
77 def get_default_cache_root():
@@ -463,6 +464,8 @@ environment_variables: Dict[str, Callable[[], Any]] = {
464 lambda: bool(int(os.getenv("VLLM_ENABLE_V1_MULTIPROCESSING", "1"))),
465 "VLLM_LOG_BATCHSIZE_INTERVAL":
466 lambda: float(os.getenv("VLLM_LOG_BATCHSIZE_INTERVAL", "-1")),
467 + "VLLM_DISABLE_COMPILE_CACHE":
468 + lambda: bool(int(os.getenv("VLLM_DISABLE_COMPILE_CACHE", "0"))),
469 }
470
471 # end-env-vars-definition